private void DeleteFilesAndDirsExcept(string fileToKeep, string dirToKeep, ITracer tracer) { // Best effort. Using the "Safe" variants does retries and swallows exceptions but // we may catch something non-obvious. try { var files = FileSystemHelpers.GetFiles(_environment.ZipTempPath, "*") .Where(p => !PathUtilityFactory.Instance.PathsEquals(p, fileToKeep)); foreach (var file in files) { FileSystemHelpers.DeleteFileSafe(file); } var dirs = FileSystemHelpers.GetDirectories(_environment.ZipTempPath) .Where(p => !PathUtilityFactory.Instance.PathsEquals(p, dirToKeep)); foreach (var dir in dirs) { FileSystemHelpers.DeleteDirectorySafe(dir); } } catch (Exception ex) { tracer.TraceError(ex, "Exception encountered during zip folder cleanup"); throw; } }
public void TracerMaxXmlFilesTest() { // Mock var path = @"x:\kudu\trace"; var tracer = new XmlTracer(path, TraceLevel.Verbose); FileSystemHelpers.Instance = GetMockFileSystem(); var total = XmlTracer.MaxXmlFiles + 10; for (int i = 0; i < total; ++i) { tracer.Trace(Guid.NewGuid().ToString(), new Dictionary <string, string>()); } var files = FileSystemHelpers.GetFiles(path, "*.xml"); Assert.Equal(total, files.Length); // wait till interval and write another trace typeof(XmlTracer).GetField("_lastCleanup", BindingFlags.Static | BindingFlags.NonPublic).SetValue(null, DateTime.MinValue); tracer.Trace(Guid.NewGuid().ToString(), new Dictionary <string, string>()); files = FileSystemHelpers.GetFiles(path, "*.xml"); Assert.True(files.Length < XmlTracer.MaxXmlFiles); }
private void EnsureMaxXmlFiles() { var now = DateTime.UtcNow; if (_lastCleanup.AddSeconds(CleanUpIntervalSecs) > now) { return; } _lastCleanup = now; try { var files = FileSystemHelpers.GetFiles(_path, "*.xml"); if (files.Length < MaxXmlFiles) { return; } foreach (var file in files.OrderBy(n => n).Take(files.Length - (MaxXmlFiles * 80) / 100)) { FileSystemHelpers.DeleteFileSafe(file); } } catch { // no-op } }
public static void PurgeBuildArtifactsIfNecessary(string sitePackagesPath, BuildArtifactType fileExtension, ITracer tracer, int totalAllowedFiles) { string extension = fileExtension.ToString().ToLowerInvariant(); IEnumerable <string> fileNames = FileSystemHelpers.GetFiles(sitePackagesPath, $"*.{extension}"); if (fileNames.Count() > totalAllowedFiles) { // Order the files in descending order of the modified date and remove the last (N - allowed zip files). var fileNamesToDelete = fileNames.OrderByDescending(fileName => FileSystemHelpers.GetLastWriteTimeUtc(fileName)).Skip(totalAllowedFiles); foreach (var fileName in fileNamesToDelete) { using (tracer.Step("Deleting outdated zip file {0}", fileName)) { try { File.Delete(fileName); } catch (Exception ex) { tracer.TraceError(ex, "Unable to delete zip file {0}", fileName); } } } } }
public static async Task BuildDotnetProject(string outputPath, string dotnetCliParams) { EnsureDotnet(); var csProjFiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: "*.csproj").ToList(); if (csProjFiles.Count == 1) { if (FileSystemHelpers.DirectoryExists(outputPath)) { FileSystemHelpers.DeleteDirectorySafe(outputPath); } var exe = new Executable("dotnet", $"build --output {outputPath} {dotnetCliParams}"); var exitCode = await exe.RunAsync(o => ColoredConsole.WriteLine(o), e => ColoredConsole.Error.WriteLine(e)); if (exitCode != 0) { throw new CliException("Error building project"); } } else { throw new CliException($"Can't determine Project to build. Expected 1 .csproj but found {csProjFiles.Count}"); } }
private void RunDeterminePrimaryScriptFileFunc(string expect, string jObjectStr, string dir) { JObject functionConfig = JObject.Parse(jObjectStr); var traceFactoryMock = new Mock <ITraceFactory>(); traceFactoryMock.Setup(tf => tf.GetTracer()).Returns(NullTracer.Instance); var functionManager = new FunctionManager(new Mock <IEnvironment>().Object, traceFactoryMock.Object); string[] functionFiles = FileSystemHelpers.GetFiles(dir, "*.*", SearchOption.TopDirectoryOnly) .Where(p => !String.Equals(Path.GetFileName(p), Constants.FunctionsConfigFile, StringComparison.OrdinalIgnoreCase)) .ToArray(); string scriptFile = (string)functionConfig["scriptFile"]; string scriptPath = string.IsNullOrEmpty(scriptFile) ? null : Path.Combine(dir, scriptFile); if ((string.IsNullOrEmpty(scriptFile) && functionFiles.Length == 0) || expect == null) { Assert.Null(functionManager.DeterminePrimaryScriptFile((string)functionConfig["scriptFile"], dir)); return; } if (!string.IsNullOrEmpty(scriptPath) && !FileSystemHelpers.FileExists(scriptPath)) { Assert.Throws <ConfigurationErrorsException>(() => functionManager.DeterminePrimaryScriptFile((string)functionConfig["scriptFile"], dir)); } else { Assert.Equal(expect, functionManager.DeterminePrimaryScriptFile((string)functionConfig["scriptFile"], dir), StringComparer.OrdinalIgnoreCase); } }
/// <summary> /// <para>Loop thru all site extension settings files under 'siteExtensionStatusRoot', check if there is any successful installation</para> /// <para>if not install to webroot, will trigger restart; if install to webroot and with applicationHost.xdt file, trigger restart.</para> /// </summary> /// <param name="siteExtensionStatusRoot">should be $ROOT\site\siteextensions</param> /// <param name="siteExtensionRoot">should be $ROOT\SiteExtensions</param> public static bool IsAnyInstallationRequireRestart(string siteExtensionStatusRoot, string siteExtensionRoot, ITracer tracer, IAnalytics analytics) { try { string[] packageDirs = FileSystemHelpers.GetDirectories(siteExtensionStatusRoot); foreach (var dir in packageDirs) { try { DirectoryInfo dirInfo = new DirectoryInfo(dir); string[] settingFile = FileSystemHelpers.GetFiles(dir, _statusSettingsFileName); foreach (var file in settingFile) { var statusSettings = new SiteExtensionStatus(siteExtensionStatusRoot, dirInfo.Name, tracer); if (statusSettings.IsSiteExtensionRequireRestart(siteExtensionRoot)) { return(true); } } } catch (Exception ex) { analytics.UnexpectedException(ex, trace: false); tracer.TraceError(ex, "Failed to query {0} under {1}, continus to check others ...", _statusSettingsFileName, dir); } } } catch (Exception ex) { analytics.UnexpectedException(ex, trace: false); tracer.TraceError(ex, "Not able to query directory under {0}", siteExtensionStatusRoot); } return(false); }
private void EnsureMaxXmlFiles() { // _lastCleanup ensures we only check (iterate) the log directory every certain period. // _cleanupPending ensures there is one cleanup thread at a time. lock (_cleanupLock) { if (_cleanupPending) { return; } var now = DateTime.UtcNow; if (_lastCleanup.AddSeconds(CleanUpIntervalSecs) > now) { return; } _lastCleanup = now; _cleanupPending = true; } try { ITracer tracer = this; using (tracer.Step("Cleanup Xml Logs")) { try { var files = FileSystemHelpers.GetFiles(_path, "*.xml"); if (files.Length < MaxXmlFiles) { return; } var toCleanup = files.Length - (MaxXmlFiles * 80) / 100; var attribs = new Dictionary <string, string> { { "totalFiles", files.Length.ToString() }, { "totalCleanup", toCleanup.ToString() } }; using (tracer.Step("Cleanup Infos", attribs)) { foreach (var file in files.OrderBy(n => n).Take(toCleanup)) { FileSystemHelpers.DeleteFileSafe(file); } } } catch (Exception ex) { tracer.TraceError(ex); } } } finally { _cleanupPending = false; } }
private string GetFunctionScriptPath(string functionName, JObject functionConfig) { var functionPath = GetFunctionPath(functionName); var functionFiles = FileSystemHelpers.GetFiles(functionPath, "*.*", SearchOption.TopDirectoryOnly) .Where(p => Path.GetFileName(p).ToLowerInvariant() != "function.json").ToArray(); return(DeterminePrimaryScriptFile(functionConfig, functionFiles)); }
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, BuildOption buildOption, string additionalPackages) { var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt); if (!FileSystemHelpers.FileExists(reqTxtFile)) { throw new CliException($"{Constants.RequirementsTxt} is not found. " + $"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing."); } var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages); if (FileSystemHelpers.DirectoryExists(packagesLocation)) { // Only update packages if checksum of requirements.txt does not match // If build option is remote, we don't need to verify if packages are in sync, as we need to delete them regardless if (buildOption != BuildOption.Remote && await ArePackagesInSync(reqTxtFile, packagesLocation)) { ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies...")); return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); } ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory"); FileSystemHelpers.DeleteDirectorySafe(packagesLocation); } FileSystemHelpers.EnsureDirectory(packagesLocation); // Only one of the remote build or build-native-deps flag can be chosen if (buildNativeDeps) { if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess()) { await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages); } else { throw new CliException("Docker is required to build native dependencies for python function apps"); } } else if (buildOption == BuildOption.Remote) { // No-ops, python packages will be resolved on the server side } else { await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation); } // No need to generate and compare .md5 when using remote build if (buildOption != BuildOption.Remote) { // Store a checksum of requirements.txt var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5"); await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile)); } return(await ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); }
private Boolean IsFolderModified(JObject fileObj, string directoryPath) { //Fetch all files in this directory string[] filePaths = FileSystemHelpers.GetFiles(directoryPath, "*"); if (filePaths != null) { foreach (string filePath in filePaths) { Console.WriteLine(filePath); //If manifest does not contain an entry for this file //It means the file was newly added //We need to scan as this is a modification JToken val = null; if (!fileObj.TryGetValue(filePath, out val)) { return(true); } //Modified time in manifest String lastModTime = (string)fileObj[filePath]; //Current modified time String currModTime = FileSystemHelpers.GetDirectoryLastWriteTimeUtc(filePath).ToString(); //If they are different //It means file has been modified after last scan if (!currModTime.Equals(lastModTime)) { return(true); } } } //Fetch all the child directories of this directory string[] direcPaths = FileSystemHelpers.GetDirectories(directoryPath); if (direcPaths != null) { //Do recursive comparison of all files in the child directories foreach (string direcPath in direcPaths) { if (IsFolderModified(fileObj, direcPath)) { return(true); } } } //No modifications found return(false); }
// Logic for this function is copied from here: // https://github.com/Azure/azure-webjobs-sdk-script/blob/dev/src/WebJobs.Script/Host/ScriptHost.cs // These two implementations must stay in sync! /// <summary> /// Determines which script should be considered the "primary" entry point script. /// </summary> /// <exception cref="ConfigurationErrorsException">Thrown if the function metadata points to an invalid script file, or no script files are present.</exception> internal string DeterminePrimaryScriptFile(JObject functionConfig, string scriptDirectory) { // First see if there is an explicit primary file indicated // in config. If so use that. string functionPrimary = null; string scriptFile = (string)functionConfig["scriptFile"]; if (!string.IsNullOrEmpty(scriptFile)) { string scriptPath = Path.Combine(scriptDirectory, scriptFile); if (!FileSystemHelpers.FileExists(scriptPath)) { TraceAndThrowError(new ConfigurationErrorsException("Invalid script file name configuration. The 'scriptFile' property is set to a file that does not exist.")); } functionPrimary = scriptPath; } else { string[] functionFiles = FileSystemHelpers.GetFiles(scriptDirectory, "*.*", SearchOption.TopDirectoryOnly) .Where(p => !String.Equals(Path.GetFileName(p), "function.json", StringComparison.OrdinalIgnoreCase)) .ToArray(); if (functionFiles.Length == 0) { TraceAndThrowError(new ConfigurationErrorsException("No function script files present.")); } if (functionFiles.Length == 1) { // if there is only a single file, that file is primary functionPrimary = functionFiles[0]; } else { // if there is a "run" file, that file is primary, // for Node, any index.js file is primary functionPrimary = functionFiles.FirstOrDefault(p => String.Equals(Path.GetFileNameWithoutExtension(p), "run", StringComparison.OrdinalIgnoreCase) || String.Equals(Path.GetFileName(p), "index.js", StringComparison.OrdinalIgnoreCase)); } } if (string.IsNullOrEmpty(functionPrimary)) { TraceAndThrowError(new ConfigurationErrorsException("Unable to determine the primary function script. Try renaming your entry point script to 'run' (or 'index' in the case of Node), " + "or alternatively you can specify the name of the entry point script explicitly by adding a 'scriptFile' property to your function metadata.")); } return(Path.GetFullPath(functionPrimary)); }
public void TracerRequestsTest(TraceLevel traceLevel, RequestInfo[] requests) { // Mock var path = @"x:\kudu\trace"; var tracer = new XmlTracer(path, traceLevel); FileSystemHelpers.Instance = GetMockFileSystem(); try { foreach (var request in requests) { Dictionary <string, string> attribs = new Dictionary <string, string> { { "type", "request" }, { "url", request.Url }, { "method", "GET" }, { "statusCode", ((int)request.StatusCode).ToString() }, }; using (tracer.Step("Incoming Request", attribs)) { tracer.Trace("Outgoing response", attribs); } } var traces = new List <RequestInfo>(); foreach (var file in FileSystemHelpers.GetFiles(path, "*s.xml").OrderBy(n => n)) { var document = XDocument.Load(FileSystemHelpers.OpenRead(file)); var trace = new RequestInfo { Url = document.Root.Attribute("url").Value }; var elapsed = document.Root.Nodes().Last(); // Assert Assert.Equal(XmlNodeType.Comment, elapsed.NodeType); Assert.Contains("duration:", ((XComment)elapsed).Value); traces.Add(trace); } // Assert Assert.Equal(requests.Where(r => r.Traced), traces, RequestInfoComparer.Instance); } catch (Exception) { FileSystemHelpers.Instance = null; throw; } }
public static string GetCsproj() { EnsureDotnet(); var csProjFiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: "*.csproj").ToList(); if (csProjFiles.Count == 1) { return(csProjFiles.First()); } else { throw new CliException($"Can't determin Project to build. Expected 1 .csproj but found {csProjFiles.Count}"); } }
// Logic for this function is copied from here: // https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script/Host/FunctionMetadataProvider.cs // These two implementations must stay in sync! /// <summary> /// Determines which script should be considered the "primary" entry point script. Returns null if Primary script file cannot be determined /// </summary> internal string DeterminePrimaryScriptFile(string scriptFile, string scriptDirectory) { // First see if there is an explicit primary file indicated // in config. If so use that. string functionPrimary = null; if (!string.IsNullOrEmpty(scriptFile)) { string scriptPath = Path.Combine(scriptDirectory, scriptFile); if (!FileSystemHelpers.FileExists(scriptPath)) { TraceAndThrowError(new ConfigurationErrorsException("Invalid script file name configuration. The 'scriptFile' property is set to a file that does not exist.")); } functionPrimary = scriptPath; } else { string[] functionFiles = FileSystemHelpers.GetFiles(scriptDirectory, "*.*", SearchOption.TopDirectoryOnly) .Where(p => !String.Equals(Path.GetFileName(p), Constants.FunctionsConfigFile, StringComparison.OrdinalIgnoreCase)) .ToArray(); if (functionFiles.Length == 0) { return(null); } if (functionFiles.Length == 1) { // if there is only a single file, that file is primary functionPrimary = functionFiles[0]; } else { // if there is a "run" file, that file is primary, // for Node, any index.js file is primary functionPrimary = functionFiles.FirstOrDefault(p => String.Equals(Path.GetFileNameWithoutExtension(p), "run", StringComparison.OrdinalIgnoreCase) || String.Equals(Path.GetFileName(p), "index.js", StringComparison.OrdinalIgnoreCase)); } } // Primary script file is not required for HttpWorker or any custom language worker if (string.IsNullOrEmpty(functionPrimary)) { return(null); } return(Path.GetFullPath(functionPrimary)); }
public static bool LooksLikePython(string siteFolder) { if (!OSDetector.IsOnWindows()) { // For Linux web apps: Rely on WEBSITE_PYTHON_VERSION environment variable to // detect if this is a python app string pythonVersion = System.Environment.GetEnvironmentVariable("WEBSITE_PYTHON_VERSION"); if (!string.IsNullOrEmpty(pythonVersion) && pythonVersion.StartsWith("3", StringComparison.OrdinalIgnoreCase)) { return(true); } return(false); } var reqsFilePath = Path.Combine(siteFolder, RequirementsFileName); if (!FileSystemHelpers.FileExists(reqsFilePath)) { return(false); } var pythonFiles = FileSystemHelpers.GetFiles(siteFolder, PythonFileExtension); if (pythonFiles.Length > 0) { return(true); } // Most Python sites will have at least a .py file in the root, but // some may not. In that case, let them opt in with the runtime.txt // file, which is used to specify the version of Python. var runtimeFilePath = Path.Combine(siteFolder, RuntimeFileName); if (FileSystemHelpers.FileExists(runtimeFilePath)) { try { var text = FileSystemHelpers.ReadAllTextFromFile(runtimeFilePath); return(text.IndexOf("python", StringComparison.OrdinalIgnoreCase) >= 0); } catch (IOException) { return(false); } } return(false); }
private static async Task DotnetTemplatesAction(string action) { var templatesLocation = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "templates"); if (!FileSystemHelpers.DirectoryExists(templatesLocation)) { throw new CliException($"Can't find templates location. Looked under '{templatesLocation}'"); } foreach (var nupkg in FileSystemHelpers.GetFiles(templatesLocation, null, null, "*.nupkg")) { var exe = new Executable("dotnet", $"new --{action} \"{nupkg}\""); await exe.RunAsync(); } }
public static bool IsAnyPendingLock(string rootPath, ITracer tracer) { bool hasPendingLock = false; try { using (tracer.Step("Checking if there is other pending installation ...")) { string[] packageDirs = FileSystemHelpers.GetDirectories(rootPath); foreach (var dir in packageDirs) { string[] lockFiles = FileSystemHelpers.GetFiles(dir, string.Format("*{0}", LockNameSuffix)); foreach (var file in lockFiles) { // If there's no file then there's no process holding onto it if (!FileSystemHelpers.FileExists(file)) { continue; } try { // If there is a file, lets see if someone has an open handle to it, or if it's // just hanging there for no reason using (FileSystemHelpers.OpenFile(file, FileMode.Open, FileAccess.Write, FileShare.Read)) { } } catch { hasPendingLock = true; break; } } if (hasPendingLock) { break; } } } } catch (Exception ex) { // no-op tracer.TraceError(ex, "Failed to check pending installation lock. Assume there is no pending lock."); } return(hasPendingLock); }
public static bool CanDotnetBuild() { EnsureDotnet(); var csProjFiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: "*.csproj").ToList(); // If the project name is extensions only then is extensions.csproj a valid csproj file if (!Path.GetFileName(Environment.CurrentDirectory).Equals("extensions")) { csProjFiles.Remove("extensions.csproj"); } if (csProjFiles.Count > 1) { throw new CliException($"Can't determine Project to build. Expected 1 .csproj but found {csProjFiles.Count}"); } return(csProjFiles.Count == 1); }
private static IEnumerable <string> GetBindings() { var functionJsonfiles = FileSystemHelpers.GetFiles(Environment.CurrentDirectory, searchPattern: Constants.FunctionJsonFileName); var bindings = new HashSet <string>(); foreach (var functionJson in functionJsonfiles) { string functionJsonContents = FileSystemHelpers.ReadAllTextFromFile(functionJson); var functionMetadata = JsonConvert.DeserializeObject <FunctionMetadata>(functionJsonContents); foreach (var binding in functionMetadata.Bindings) { bindings.Add(binding.Type.ToLower()); } } return(bindings); }
protected override void OnLockRelease() { // if installed failed, when release lock, we should also remove empty folder as well base.OnLockRelease(); try { string folder = Path.GetDirectoryName(_path); if (FileSystemHelpers.DirectoryExists(folder) && FileSystemHelpers.GetFiles(folder, "*").Length == 0) { FileSystemHelpers.DeleteDirectorySafe(folder); } } catch { // no-op } }
/// <summary> /// Update the status and the detailed status of the job. /// The status is the status of one of the instances, /// The detailed status contains status for all instances and looks like: /// /// aabbcc - Running /// 112233 - PendingRestart /// /// </summary> private void UpdateDetailedStatus(ContinuousJob job, string jobsSpecificDataPath) { string instanceId = InstanceIdUtility.GetShortInstanceId(); string[] statusFiles = FileSystemHelpers.GetFiles(jobsSpecificDataPath, StatusFilesSearchPattern); if (statusFiles.Length <= 0) { // If no status files exist update to default values string status = ContinuousJobStatus.Initializing.Status; job.DetailedStatus = instanceId + " - " + status; job.Status = status; return; } string lastStatus = null; var stringBuilder = new StringBuilder(); foreach (string statusFile in statusFiles) { // Extract instance id from the file name int statusFileNameIndex = statusFile.IndexOf(ContinuousJobStatus.FileNamePrefix, StringComparison.OrdinalIgnoreCase); string statusFileInstanceId = statusFile.Substring(statusFileNameIndex + ContinuousJobStatus.FileNamePrefix.Length); // Try to delete file, it'll be deleted if no one holds a lock on it // That way we know the status file is obsolete if (String.Equals(statusFileInstanceId, instanceId, StringComparison.OrdinalIgnoreCase) || !TryDelete(statusFile)) { // If we couldn't delete the file, we know it holds the status of an actual instance holding it var continuousJobStatus = GetStatus <ContinuousJobStatus>(statusFile) ?? ContinuousJobStatus.Initializing; stringBuilder.AppendLine(statusFileInstanceId + " - " + continuousJobStatus.Status); if (lastStatus == null || !ContinuousJobStatus.InactiveInstance.Equals(continuousJobStatus)) { lastStatus = continuousJobStatus.Status; } } } // Job status will only show one of the statuses job.Status = lastStatus; job.DetailedStatus = stringBuilder.ToString(); }
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, string additionalPackages) { var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt); if (!FileSystemHelpers.FileExists(reqTxtFile)) { throw new CliException($"{Constants.RequirementsTxt} is not found. " + $"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing."); } var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages); if (FileSystemHelpers.DirectoryExists(packagesLocation)) { // Only update packages if checksum of requirements.txt does not match or a sync is forced if (await ArePackagesInSync(reqTxtFile, packagesLocation)) { ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies...")); return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); } ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory"); FileSystemHelpers.DeleteDirectorySafe(Path.Combine(functionAppRoot, Constants.ExternalPythonPackages)); } FileSystemHelpers.EnsureDirectory(packagesLocation); if (buildNativeDeps) { if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess()) { await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages); } else { throw new CliException("Docker is required to build native dependencies for python function apps"); } } else { await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation); } // Store a checksum of requirements.txt var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5"); await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile)); return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot)); }
private static StreamContent CreateZip(string path) { var memoryStream = new MemoryStream(); using (var zip = new ZipArchive(memoryStream, ZipArchiveMode.Create, leaveOpen: true)) { foreach (var fileName in FileSystemHelpers.GetFiles(path, new[] { ".git", ".vscode" }, new[] { ".gitignore", "appsettings.json", "local.settings.json", "project.lock.json" })) { zip.AddFile(fileName, fileName, path); } } memoryStream.Seek(0, SeekOrigin.Begin); var content = new StreamContent(memoryStream); content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); return(content); }
// Logic for this function is copied from here // https://github.com/Azure/azure-webjobs-sdk-script/blob/e0a783e882dd8680bf23e3c8818fb9638071c21d/src/WebJobs.Script/Config/ScriptHost.cs#L113-L150 private string GetFunctionScriptPath(string functionName, JObject functionInfo) { var functionPath = GetFunctionPath(functionName); var functionFiles = FileSystemHelpers.GetFiles(functionPath, "*.*", SearchOption.TopDirectoryOnly) .Where(p => Path.GetFileName(p).ToLowerInvariant() != "function.json").ToArray(); if (functionFiles.Length == 0) { return(functionPath); } else if (functionFiles.Length == 1) { // if there is only a single file, that file is primary return(functionFiles[0]); } else { // if there is a "run" file, that file is primary string functionPrimary = null; functionPrimary = functionFiles.FirstOrDefault(p => Path.GetFileNameWithoutExtension(p).ToLowerInvariant() == "run"); if (string.IsNullOrEmpty(functionPrimary)) { // for Node, any index.js file is primary functionPrimary = functionFiles.FirstOrDefault(p => Path.GetFileName(p).ToLowerInvariant() == "index.js"); if (string.IsNullOrEmpty(functionPrimary)) { // finally, if there is an explicit primary file indicated // in config, use it JToken token = functionInfo["source"]; if (token != null) { string sourceFileName = (string)token; functionPrimary = Path.Combine(functionPath, sourceFileName); } } } if (string.IsNullOrEmpty(functionPrimary)) { // TODO: should this be an error? return(functionPath); } return(functionPrimary); } }
public static bool IsAnyPendingLock(string rootPath) { bool hasPendingLock = false; try { string[] packageDirs = FileSystemHelpers.GetDirectories(rootPath); foreach (var dir in packageDirs) { string[] lockFiles = FileSystemHelpers.GetFiles(dir, string.Format("*{0}", LockNameSuffix)); foreach (var file in lockFiles) { // If there's no file then there's no process holding onto it if (!FileSystemHelpers.FileExists(file)) { continue; } try { // If there is a file, lets see if someone has an open handle to it, or if it's // just hanging there for no reason using (FileSystemHelpers.OpenFile(file, FileMode.Open, FileAccess.Write, FileShare.Read)) { } } catch { hasPendingLock = true; break; } } if (hasPendingLock) { break; } } } catch { // no-op } return(hasPendingLock); }
public static IEnumerable <string> GetLocalFiles(string path, GitIgnoreParser ignoreParser = null, bool returnIgnored = false) { var ignoredDirectories = new[] { ".git", ".vscode" }; var ignoredFiles = new[] { ".funcignore", ".gitignore", "appsettings.json", "local.settings.json", "project.lock.json" }; foreach (var file in FileSystemHelpers.GetFiles(path, ignoredDirectories, ignoredFiles)) { if (preCondition(file)) { yield return(file); } } bool preCondition(string file) { var fileName = file.Replace(path, string.Empty).Trim(Path.DirectorySeparatorChar).Replace("\\", "/"); return((returnIgnored ? ignoreParser?.Denies(fileName) : ignoreParser?.Accepts(fileName)) ?? true); } }
private void ModifyManifestFile(JObject fileObj, string directoryPath) { //Get all files in this directory string[] filePaths = FileSystemHelpers.GetFiles(directoryPath, "*"); foreach (string filePath in filePaths) { //Get last modified timestamp of this file String timeString = FileSystemHelpers.GetDirectoryLastWriteTimeUtc(filePath).ToString(); //Add it as an entry into the manifest fileObj.Add(filePath, timeString); } //Get all child directories of this directory string[] direcPaths = FileSystemHelpers.GetDirectories(directoryPath); //Do a recursive call to add files of child directories to manifest foreach (string direcPath in direcPaths) { ModifyManifestFile(fileObj, direcPath); } }
public static bool LooksLikePython(string siteFolder) { var reqsFilePath = Path.Combine(siteFolder, RequirementsFileName); if (!FileSystemHelpers.FileExists(reqsFilePath)) { return(false); } var pythonFiles = FileSystemHelpers.GetFiles(siteFolder, PythonFileExtension); if (pythonFiles.Length > 0) { return(true); } // Most Python sites will have at least a .py file in the root, but // some may not. In that case, let them opt in with the runtime.txt // file, which is used to specify the version of Python. var runtimeFilePath = Path.Combine(siteFolder, RuntimeFileName); if (FileSystemHelpers.FileExists(runtimeFilePath)) { try { var text = FileSystemHelpers.ReadAllTextFromFile(runtimeFilePath); return(text.IndexOf("python", StringComparison.OrdinalIgnoreCase) >= 0); } catch (IOException) { return(false); } } return(false); }
public static void PurgeZipsIfNecessary(string sitePackagesPath, ITracer tracer, int totalAllowedZips) { IEnumerable <string> zipFiles = FileSystemHelpers.GetFiles(sitePackagesPath, "*.zip"); if (zipFiles.Count() > totalAllowedZips) { // Order the files in descending order of the modified date and remove the last (N - allowed zip files). var fileNamesToDelete = zipFiles.OrderByDescending(fileName => FileSystemHelpers.GetLastWriteTimeUtc(fileName)).Skip(totalAllowedZips); foreach (var fileName in fileNamesToDelete) { using (tracer.Step("Deleting outdated zip file {0}", fileName)) { try { File.Delete(fileName); } catch (Exception ex) { tracer.TraceError(ex, "Unable to delete zip file {0}", fileName); } } } } }