internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, string additionalPackages)
{
var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt);
if (!FileSystemHelpers.FileExists(reqTxtFile))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages);
if (FileSystemHelpers.DirectoryExists(packagesLocation))
{
// Only update packages if checksum of requirements.txt does not match or a sync is forced
if (await ArePackagesInSync(reqTxtFile, packagesLocation))
{
ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies..."));
return(ZipHelper.CreateZip(files.Concat(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory");
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(functionAppRoot, Constants.ExternalPythonPackages));
}
FileSystemHelpers.EnsureDirectory(packagesLocation);
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages);
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else
{
await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation);
}
// Store a checksum of requirements.txt
var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5");
await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile));
return(ZipHelper.CreateZip(files.Concat(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
protected override void OnLockRelease()
{
// if installed failed, when release lock, we should also remove empty folder as well
base.OnLockRelease();
try
{
string folder = Path.GetDirectoryName(_path);
if (FileSystemHelpers.DirectoryExists(folder) && FileSystemHelpers.GetFiles(folder, "*").Length == 0)
{
FileSystemHelpers.DeleteDirectorySafe(folder);
}
}
catch
{
// no-op
}
}
private static void CleanBuild(ITracer tracer, string buildTempPath)
{
if (buildTempPath != null)
{
using (tracer.Step("Cleaning up temp files"))
{
try
{
FileSystemHelpers.DeleteDirectorySafe(buildTempPath);
}
catch (Exception ex)
{
tracer.TraceError(ex);
}
}
}
}
public static async Task <bool> BuildDotnetProject(string outputPath, string dotnetCliParams, bool showOutput = true)
{
if (FileSystemHelpers.DirectoryExists(outputPath))
{
FileSystemHelpers.DeleteDirectorySafe(outputPath);
}
var exe = new Executable("dotnet", $"build --output {outputPath} {dotnetCliParams}");
var exitCode = showOutput
? await exe.RunAsync(o => ColoredConsole.WriteLine(o), e => ColoredConsole.Error.WriteLine(e))
: await exe.RunAsync();
if (exitCode != 0)
{
throw new CliException("Error building project");
}
return(true);
}
public async Task <bool> UninstallExtension(string id)
{
ITracer tracer = _traceFactory.GetTracer();
string installationDirectory = GetInstallationDirectory(id);
SiteExtensionInfo info = await GetLocalExtension(id, checkLatest : false);
if (info == null || !FileSystemHelpers.DirectoryExists(info.LocalPath))
{
tracer.TraceError("Site extension {0} not found.", id);
throw new DirectoryNotFoundException(installationDirectory);
}
var externalCommandFactory = new ExternalCommandFactory(_environment, _settings, installationDirectory);
string uninstallScript = Path.Combine(installationDirectory, _uninstallScriptName);
if (FileSystemHelpers.FileExists(uninstallScript))
{
using (tracer.Step("Execute uninstall.cmd"))
{
OperationManager.Attempt(() =>
{
Executable exe = externalCommandFactory.BuildCommandExecutable(uninstallScript,
installationDirectory,
_settings.GetCommandIdleTimeout(), NullLogger.Instance);
exe.ExecuteWithProgressWriter(NullLogger.Instance, _traceFactory.GetTracer(), String.Empty);
});
}
}
using (tracer.Step("Remove site extension job"))
{
OperationManager.Attempt(() => CleanupSiteExtensionJobs(id));
}
using (tracer.Step("Delete site extension package and directory"))
{
OperationManager.Attempt(() => FileSystemHelpers.DeleteFileSafe(GetNuGetPackageFile(info.Id, info.Version)));
OperationManager.Attempt(() => FileSystemHelpers.DeleteDirectorySafe(installationDirectory));
}
return(await GetLocalExtension(id, checkLatest : false) == null);
}
private void CacheJobBinaries(IJobLogger logger)
{
if (WorkingDirectory != null)
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
WorkingDirectory = tempJobInstancePath;
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
WorkingDirectory = null;
}
}
private void AddOryxBuildCommand(StringBuilder args, DeploymentContext context, string source, string destination)
{
// If it is express build, we don't directly need to write to /home/site/wwwroot
// So, we build into a different directory to avoid overlap
// Additionally, we didn't run kudusync, and can just build directly from repository path
if (Flags == BuildOptimizationsFlags.UseExpressBuild)
{
source = context.RepositoryPath;
destination = OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup;
// It is important to clean and recreate the directory to make sure no overwrite occurs
if (FileSystemHelpers.DirectoryExists(destination))
{
FileSystemHelpers.DeleteDirectorySafe(destination);
}
FileSystemHelpers.EnsureDirectory(destination);
}
OryxArgumentsHelper.AddOryxBuildCommand(args, source, destination);
}
private IPackage InstallExtension(IPackage package, string installationDirectory)
{
try
{
if (FileSystemHelpers.DirectoryExists(installationDirectory))
{
FileSystemHelpers.DeleteDirectorySafe(installationDirectory);
}
foreach (IPackageFile file in package.GetContentFiles())
{
// It is necessary to place applicationHost.xdt under site extension root.
string contentFilePath = file.Path.Substring("content/".Length);
string fullPath = Path.Combine(installationDirectory, contentFilePath);
FileSystemHelpers.CreateDirectory(Path.GetDirectoryName(fullPath));
using (Stream writeStream = FileSystemHelpers.OpenWrite(fullPath), readStream = file.GetStream())
{
OperationManager.Attempt(() => readStream.CopyTo(writeStream));
}
}
// If there is no xdt file, generate default.
GenerateApplicationHostXdt(installationDirectory, '/' + package.Id, isPreInstalled: false);
// Copy nupkg file for package list/lookup
FileSystemHelpers.CreateDirectory(installationDirectory);
string packageFilePath = Path.Combine(installationDirectory,
String.Format("{0}.{1}.nupkg", package.Id, package.Version));
using (
Stream readStream = package.GetStream(), writeStream = FileSystemHelpers.OpenWrite(packageFilePath))
{
OperationManager.Attempt(() => readStream.CopyTo(writeStream));
}
}
catch (Exception ex)
{
ITracer tracer = _traceFactory.GetTracer();
tracer.TraceError(ex);
FileSystemHelpers.DeleteDirectorySafe(installationDirectory);
return(null);
}
return(_localRepository.FindPackage(package.Id));
}
private static bool IsLockValid()
{
if (!FileSystemHelpers.FileExists(locksPath + "/deployment/info.lock"))
{
return(false);
}
// No need to serialize lock expiry object again until the
// lock expiry period, we would use local cache instead
// At this point we have already checked for the folder presence
// hence to avoid the I/O, don't serialize the lock info until
// folder is cleaned up
if (!string.IsNullOrEmpty(LockExpiry))
{
if (Convert.ToDateTime(LockExpiry) > DateTime.Now)
{
return(true);
}
}
var lockInfo = JObject.Parse(File.ReadAllText(locksPath + "/deployment/info.lock"));
var workerId = lockInfo[$"heldByWorker"].ToString();
var expStr = lockInfo[$"lockExpiry"].ToString();
//Should never have null expiry
if (string.IsNullOrEmpty(expStr))
{
Console.WriteLine("IsLockValid - Null Expiry | This is Bad");
FileSystemHelpers.DeleteDirectorySafe(locksPath + "/deployment");
return(false);
}
var exp = Convert.ToDateTime(expStr);
LockExpiry = expStr;
if (exp > DateTime.UtcNow)
{
return(true);
}
//Console.WriteLine("IsLockValid - Lock is Past expiry - Deleting Lock Dir");
FileSystemHelpers.DeleteDirectorySafe(locksPath + "/deployment");
return(false);
}
/// <summary>
/// Handle both file and folder deletion, and ignore not found exception
/// </summary>
private void HandlingDeletion(OneDriveModel.OneDriveChange change, string wwwroot)
{
if (!change.IsDeleted)
{
return;
}
string fullPath = GetDestinationPath(wwwroot, change.Path);
if (fullPath == null)
{
TraceMessage("Ignore folder {0}", change.Path);
return;
}
try
{
if (FileSystemHelpers.FileExists(fullPath))
{
FileSystemHelpers.DeleteFile(fullPath);
TraceMessage("Deleted file {0}", fullPath);
LogMessage(Resources.OneDriveDeletedFile, fullPath);
}
else if (FileSystemHelpers.DirectoryExists(fullPath))
{
FileSystemHelpers.DeleteDirectorySafe(fullPath, ignoreErrors: false);
TraceMessage("Deleted directory {0}", fullPath);
LogMessage(Resources.OneDriveDeletedDirectory, fullPath);
}
else
{
TraceMessage("Not found: {0}. Unknown item type, skip deletion!", fullPath);
}
}
catch (DirectoryNotFoundException)
{
TraceMessage("Directory Not found: {0}. Skip deletion!", fullPath);
}
catch (FileNotFoundException)
{
TraceMessage("File Not found: {0}. Skip deletion!", fullPath);
}
}
public void Delete(string id)
{
string path = Path.Combine(_environment.DeploymentsPath, id);
_statusLock.LockOperation(() =>
{
FileSystemHelpers.DeleteDirectorySafe(path, ignoreErrors: true);
// Used for ETAG
if (_fileSystem.File.Exists(_activeFile))
{
_fileSystem.File.SetLastWriteTimeUtc(_activeFile, DateTime.UtcNow);
}
else
{
_fileSystem.File.WriteAllText(_activeFile, String.Empty);
}
}, LockTimeout);
}
public static void CleanupDeletedJobs(IEnumerable <string> existingJobs, string jobsDataPath, ITracer tracer)
{
DirectoryInfoBase jobsDataDirectory = FileSystemHelpers.DirectoryInfoFromDirectoryName(jobsDataPath);
if (jobsDataDirectory.Exists)
{
DirectoryInfoBase[] jobDataDirectories = jobsDataDirectory.GetDirectories("*", SearchOption.TopDirectoryOnly);
IEnumerable <string> allJobDataDirectories = jobDataDirectories.Select(j => j.Name);
IEnumerable <string> directoriesToRemove = allJobDataDirectories.Except(existingJobs, StringComparer.OrdinalIgnoreCase);
foreach (string directoryToRemove in directoriesToRemove)
{
using (tracer.Step("CleanupDeletedJobs"))
{
tracer.Trace("Removed job data path as the job was already deleted: " + directoryToRemove);
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(jobsDataPath, directoryToRemove));
}
}
}
}
public void Delete()
{
// Fail if a deployment is in progress
if (_deploymentLock.IsHeld)
{
HttpResponseMessage response = Request.CreateErrorResponse(HttpStatusCode.Conflict, Resources.Error_DeploymentInProgess);
throw new HttpResponseException(response);
}
using (_tracer.Step("Deleting deployment cache"))
{
// Delete the deployment cache
FileSystemHelpers.DeleteDirectorySafe(_environment.DeploymentCachePath);
}
using (_tracer.Step("Deleting repository"))
{
// Delete the repository
FileSystemHelpers.DeleteDirectorySafe(_environment.DeploymentRepositoryPath);
}
}
public async Task RemoveStatus()
{
try
{
string dirName = Path.GetDirectoryName(_filePath);
if (FileSystemHelpers.DirectoryExists(dirName))
{
FileSystemHelpers.DeleteDirectoryContentsSafe(dirName);
// call DeleteDirectorySafe directly would sometime causing "Access denied" on folder
// work-around: remove content and wait briefly before delete folder
await Task.Delay(300);
FileSystemHelpers.DeleteDirectorySafe(dirName);
}
}
catch (Exception ex)
{
// no-op
_tracer.TraceError(ex);
}
}
public static void Start()
{
if (!Kudu.Core.Environment.IsAzureEnvironment() ||
!OSDetector.IsOnWindows())
{
return;
}
if (Interlocked.Exchange(ref _running, 1) != 0)
{
return;
}
// Cleanup Sentinel on Startup
OperationManager.SafeExecute(() =>
{
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(PathUtilityFactory.Instance.ResolveLocalSitePath(), @"ShutdownSentinel"));
});
ThreadPool.QueueUserWorkItem(BackgroundTaskProc, new ManualResetEvent(initialState: false));
}
public static TestRepository Clone(string repositoryPath, string source, bool createDirectory = false)
{
// Gets full path in case path is relative
repositoryPath = GetRepositoryPath(repositoryPath);
// Make sure the directory is empty
FileSystemHelpers.DeleteDirectorySafe(repositoryPath);
Executable gitExe = GetGitExe(repositoryPath);
if (createDirectory)
{
var result = gitExe.Execute("clone \"{0}\"", source);
// Cloning into '{0}'...
var m = Regex.Match(result.Item1, @"Cloning\s+into\s+\'?(\w+)'?\.*", RegexOptions.IgnoreCase);
string folderName = m.Groups[1].Value;
return(new TestRepository(Path.Combine(repositoryPath, folderName)));
}
gitExe.Execute("clone \"{0}\" .", source);
return(new TestRepository(repositoryPath));
}
/// <summary>
/// <para>Check if there is damanged leftover data</para>
/// <para>If there is leftover data, will try to cleanup.</para>
/// <para>If fail to cleanup, will move leftover data to Temp folder</para>
/// </summary>
private static void EnsureInstallationEnviroment(string installationDir, ITracer tracer)
{
// folder is there but nupkg is gone, means previous uninstallation must encounter some error
bool isInstalledPackageBroken = FileSystemHelpers.DirectoryExists(installationDir) && FileSystemHelpers.GetFiles(installationDir, "*.nupkg").Length == 0;
if (!isInstalledPackageBroken)
{
return;
}
using (tracer.Step("There was leftover data from previous uninstallation. Trying to cleanup now."))
{
try
{
OperationManager.Attempt(() => FileSystemHelpers.DeleteDirectorySafe(installationDir, ignoreErrors: false));
return;
}
catch (Exception ex)
{
tracer.TraceError(ex);
}
FileSystemHelpers.EnsureDirectory(_toBeDeletedDirectoryPath);
DirectoryInfo dirInfo = new DirectoryInfo(installationDir);
string tmpFoder = Path.Combine(
_toBeDeletedDirectoryPath,
string.Format(CultureInfo.InvariantCulture, "{0}-{1}", dirInfo.Name, Guid.NewGuid().ToString("N").Substring(0, 8)));
using (tracer.Step("Failed to cleanup. Moving leftover data to {0}", tmpFoder))
{
// if failed, let exception bubble up to trigger bad request
OperationManager.Attempt(() => FileSystemHelpers.MoveDirectory(installationDir, tmpFoder));
}
}
FileSystemHelpers.DeleteDirectoryContentsSafe(_toBeDeletedDirectoryPath);
}
private static bool IsLockValid()
{
//Console.WriteLine("IsLockValid - InfoFileExists "+FileSystemHelpers.FileExists(locksPath+"/deployment/info.lock"));
if (!FileSystemHelpers.FileExists(locksPath + "/deployment/info.lock"))
{
return(false);
}
var lockInfo = JObject.Parse(File.ReadAllText(locksPath + "/deployment/info.lock"));
//Console.WriteLine(lockInfo);
var workerId = lockInfo[$"heldByWorker"].ToString();
var expStr = lockInfo[$"lockExpiry"].ToString();
//Console.WriteLine("IsLockValid - LockExpiry "+expStr);
//Console.WriteLine("IsLockValid - HeldByWorker "+workerId);
//Should never have null expiry
if (string.IsNullOrEmpty(expStr))
{
Console.WriteLine("IsLockValid - Null Expiry | This is Bad");
FileSystemHelpers.DeleteDirectorySafe(locksPath + "/deployment");
return(false);
}
var exp = Convert.ToDateTime(expStr.ToString());
if (exp > DateTime.UtcNow)
{
//Console.WriteLine("Expiry Time - "+exp);
//Console.WriteLine("IsLockValid - "+DateTime.UtcNow);
//Console.WriteLine("IsLockValid - Within 5 min expiry");
return(true);
}
//Console.WriteLine("IsLockValid - Lock is Past expiry - Deleting Lock Dir");
FileSystemHelpers.DeleteDirectorySafe(locksPath + "/deployment");
return(false);
}
public static void PurgeOldDeploymentsIfNecessary(string deploymentsPath, ITracer tracer, int totalAllowedDeployments)
{
IEnumerable <string> deploymentNames = FileSystemHelpers.GetDirectories(deploymentsPath);
if (deploymentNames.Count() > totalAllowedDeployments)
{
// Order the files in descending order of the modified date and remove the last (N - allowed zip files).
var deploymentsToDelete = deploymentNames.OrderByDescending(fileName => FileSystemHelpers.GetLastWriteTimeUtc(fileName)).Skip(totalAllowedDeployments);
foreach (var deploymentName in deploymentsToDelete)
{
using (tracer.Step("Purging old deployment {0}", deploymentName))
{
try
{
FileSystemHelpers.DeleteDirectorySafe(deploymentName);
}
catch (Exception ex)
{
tracer.TraceError(ex, "Unable to delete deployment {0}", deploymentName);
}
}
}
}
}
internal static async Task <Stream> GetPythonDeploymentPackage(IEnumerable <string> files, string functionAppRoot, bool buildNativeDeps, BuildOption buildOption, string additionalPackages)
{
var reqTxtFile = Path.Combine(functionAppRoot, Constants.RequirementsTxt);
if (!FileSystemHelpers.FileExists(reqTxtFile))
{
throw new CliException($"{Constants.RequirementsTxt} is not found. " +
$"{Constants.RequirementsTxt} is required for python function apps. Please make sure to generate one before publishing.");
}
var packagesLocation = Path.Combine(functionAppRoot, Constants.ExternalPythonPackages);
if (FileSystemHelpers.DirectoryExists(packagesLocation))
{
// Only update packages if checksum of requirements.txt does not match
// If build option is remote, we don't need to verify if packages are in sync, as we need to delete them regardless
if (buildOption != BuildOption.Remote && await ArePackagesInSync(reqTxtFile, packagesLocation))
{
ColoredConsole.WriteLine(Yellow($"Directory {Constants.ExternalPythonPackages} already in sync with {Constants.RequirementsTxt}. Skipping restoring dependencies..."));
return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
ColoredConsole.WriteLine($"Deleting the old {Constants.ExternalPythonPackages} directory");
FileSystemHelpers.DeleteDirectorySafe(packagesLocation);
}
FileSystemHelpers.EnsureDirectory(packagesLocation);
// Only one of the remote build or build-native-deps flag can be chosen
if (buildNativeDeps && buildOption == BuildOption.Remote)
{
throw new CliException("Cannot perform '--build-native-deps' along with '--build remote'");
}
if (buildNativeDeps)
{
if (CommandChecker.CommandExists("docker") && await DockerHelpers.VerifyDockerAccess())
{
await RestorePythonRequirementsDocker(functionAppRoot, packagesLocation, additionalPackages);
}
else
{
throw new CliException("Docker is required to build native dependencies for python function apps");
}
}
else if (buildOption == BuildOption.Remote)
{
// No-ops, python packages will be resolved on the server side
}
else
{
await RestorePythonRequirementsPackapp(functionAppRoot, packagesLocation);
}
// No need to generate and compare .md5 when using remote build
if (buildOption != BuildOption.Remote)
{
// Store a checksum of requirements.txt
var md5FilePath = Path.Combine(packagesLocation, $"{Constants.RequirementsTxt}.md5");
await FileSystemHelpers.WriteAllTextToFileAsync(md5FilePath, SecurityHelpers.CalculateMd5(reqTxtFile));
}
return(ZipHelper.CreateZip(files.Union(FileSystemHelpers.GetFiles(packagesLocation)), functionAppRoot));
}
public static async Task UpdateLocalPackage(string siteExntentionsRootPath, string packageId, string packageVersion, string destinationFolder, string pathToLocalCopyOfNupkg, ITracer tracer)
{
tracer.Trace("Performing incremental package update for {0}", packageId);
using (var client = new HttpClient())
{
var uri = new Uri(String.Format("https://www.nuget.org/api/v2/package/{0}/{1}", packageId, packageVersion));
var response = await client.GetAsync(uri);
using (Stream newPackageStream = await response.Content.ReadAsStreamAsync())
{
// update file
var localPackage = (await FeedExtensionsV2.SearchLocalRepo(siteExntentionsRootPath, packageId)).FirstOrDefault();
if (localPackage == null)
{
throw new FileNotFoundException(string.Format(CultureInfo.InvariantCulture, "Package {0} not found from local repo.", packageId));
}
string nupkgFile = Directory.GetFiles(Path.Combine(siteExntentionsRootPath, packageId), "*.nupkg", SearchOption.TopDirectoryOnly).FirstOrDefault();
using (ZipFile oldPackageZip = ZipFile.Read(nupkgFile))
using (ZipFile newPackageZip = ZipFile.Read(newPackageStream))
{
// we only care about stuff under "content" folder
IEnumerable <ZipEntry> oldContentEntries = oldPackageZip.Entries.Where(e => e.FileName.StartsWith(@"content/", StringComparison.InvariantCultureIgnoreCase));
IEnumerable <ZipEntry> newContentEntries = newPackageZip.Entries.Where(e => e.FileName.StartsWith(@"content/", StringComparison.InvariantCultureIgnoreCase));
List <ZipEntry> filesNeedToUpdate = new List <ZipEntry>();
Dictionary <string, ZipEntry> indexedOldFiles = new Dictionary <string, ZipEntry>();
foreach (var item in oldContentEntries)
{
indexedOldFiles.Add(item.FileName.ToLowerInvariant(), item);
}
foreach (var newEntry in newContentEntries)
{
var fileName = newEntry.FileName.ToLowerInvariant();
if (indexedOldFiles.ContainsKey(fileName))
{
// file name existed, only update if file has been touched
ZipEntry oldEntry = indexedOldFiles[fileName];
if (oldEntry.LastModified != newEntry.LastModified)
{
filesNeedToUpdate.Add(newEntry);
}
// remove from old index files buffer, the rest will be files that need to be deleted
indexedOldFiles.Remove(fileName);
}
else
{
// new files
filesNeedToUpdate.Add(newEntry);
}
}
int substringStartIndex = @"content/".Length;
foreach (var entry in filesNeedToUpdate)
{
string entryFileName = Uri.UnescapeDataString(entry.FileName);
string fullPath = Path.Combine(destinationFolder, entryFileName.Substring(substringStartIndex));
if (entry.IsDirectory)
{
using (tracer.Step("Ensure directory: {0}", fullPath))
{
FileSystemHelpers.EnsureDirectory(fullPath.Replace('/', '\\'));
}
continue;
}
using (tracer.Step("Adding/Updating file: {0}", fullPath))
{
FileSystemHelpers.EnsureDirectory(Path.GetDirectoryName(fullPath));
using (Stream writeStream = FileSystemHelpers.OpenWrite(fullPath))
{
// reset length of file stream
writeStream.SetLength(0);
// let the thread go with itself, so that once file finishes writing, doesn't need to request thread context from main thread
await entry.OpenReader().CopyToAsync(writeStream).ConfigureAwait(false);
}
}
}
foreach (var entry in indexedOldFiles.Values)
{
string entryFileName = Uri.UnescapeDataString(entry.FileName);
string fullPath = Path.Combine(destinationFolder, entryFileName.Substring(substringStartIndex));
if (entry.IsDirectory)
{
// in case the two zip file was created from different tool. some tool will include folder as seperate entry, some don`t.
// to be sure that folder is meant to be deleted, double check there is no files under it
var entryNameInLower = entryFileName.ToLower();
if (!string.Equals(destinationFolder, fullPath, StringComparison.OrdinalIgnoreCase) &&
newContentEntries.FirstOrDefault(e => e.FileName.ToLowerInvariant().StartsWith(entryNameInLower)) == null)
{
using (tracer.Step("Deleting directory: {0}", fullPath))
{
FileSystemHelpers.DeleteDirectorySafe(fullPath);
}
}
continue;
}
using (tracer.Step("Deleting file: {0}", fullPath))
{
FileSystemHelpers.DeleteFileSafe(fullPath);
}
}
}
// update nupkg
newPackageStream.Position = 0;
using (tracer.Step("Updating nupkg file."))
{
WriteStreamToFile(newPackageStream, pathToLocalCopyOfNupkg);
if (!packageVersion.Equals(localPackage.Version))
{
using (tracer.Step("New package has difference version {0} from old package {1}. Remove old nupkg file.", packageVersion, localPackage.Version))
{
// if version is difference, nupkg file name will be difference. will need to clean up the old one.
var oldNupkg = pathToLocalCopyOfNupkg.Replace(
string.Format(CultureInfo.InvariantCulture, "{0}.{1}.nupkg", packageId, packageVersion),
string.Format(CultureInfo.InvariantCulture, "{0}.{1}.nupkg", localPackage.Id, localPackage.Version));
FileSystemHelpers.DeleteFileSafe(oldNupkg);
}
}
}
}
}
}
/// <summary>
/// <para>1. Download package</para>
/// <para>2. Generate xdt file if not exist</para>
/// <para>3. Deploy site extension job</para>
/// <para>4. Execute install.cmd if exist</para>
/// </summary>
private async Task <UIPackageMetadata> InstallExtension(UIPackageMetadata package, string installationDirectory, SourceRepository remoteRepo, SiteExtensionInfo.SiteExtensionType type, ITracer tracer, string installationArgs)
{
try
{
EnsureInstallationEnviroment(installationDirectory, tracer);
string packageLocalFilePath = GetNuGetPackageFile(package.Identity.Id, package.Identity.Version.ToNormalizedString());
bool packageExisted = FileSystemHelpers.DirectoryExists(installationDirectory);
using (tracer.Step("Download site extension: {0}", package.Identity))
{
string extractPath = installationDirectory;
if (SiteExtensionInfo.SiteExtensionType.WebRoot == type)
{
extractPath = _environment.WebRootPath;
FileSystemHelpers.EnsureDirectory(extractPath);
}
// Copy/update content folder
// Copy/update nupkg file for package list/lookup
if (packageExisted)
{
await remoteRepo.UpdateLocalPackage(_localRepository, package.Identity, extractPath, packageLocalFilePath, tracer);
}
else
{
FileSystemHelpers.EnsureDirectory(installationDirectory);
await remoteRepo.DownloadPackageToFolder(package.Identity, extractPath, packageLocalFilePath);
}
if (SiteExtensionInfo.SiteExtensionType.WebRoot == type)
{
// if install to WebRoot, check if there is any xdt or scmXdt file come with package
// if there is, move it to site extension folder
string xdtFile = Path.Combine(extractPath, Constants.ApplicationHostXdtFileName);
string scmXdtFile = Path.Combine(extractPath, Constants.ScmApplicationHostXdtFileName);
bool findXdts = false;
if (FileSystemHelpers.FileExists(xdtFile))
{
tracer.Trace("Use xdt file from package.");
string newXdtFile = Path.Combine(installationDirectory, Constants.ApplicationHostXdtFileName);
tracer.Trace("Moving {0} to {1}", xdtFile, newXdtFile);
FileSystemHelpers.MoveFile(xdtFile, newXdtFile);
findXdts = true;
}
// if the siteextension applies to both main site and the scm site
if (FileSystemHelpers.FileExists(scmXdtFile))
{
tracer.Trace("Use scmXdt file from package.");
string newScmXdtFile = Path.Combine(installationDirectory, Constants.ScmApplicationHostXdtFileName);
tracer.Trace("Moving {0} to {1}", scmXdtFile, newScmXdtFile);
FileSystemHelpers.MoveFile(scmXdtFile, newScmXdtFile);
findXdts = true;
}
if (!findXdts)
{
tracer.Trace("No xdt file come with package.");
}
}
}
// ignore below action if we install packge to wwwroot
if (SiteExtensionInfo.SiteExtensionType.WebRoot != type)
{
using (tracer.Step("Check if applicationHost.xdt or scmApplicationHost.xdt file existed."))
{
GenerateDefaultScmApplicationHostXdt(installationDirectory, '/' + package.Identity.Id, tracer: tracer);
}
using (tracer.Step("Trigger site extension job"))
{
OperationManager.Attempt(() => DeploySiteExtensionJobs(package.Identity.Id));
}
var externalCommandFactory = new ExternalCommandFactory(_environment, _settings, installationDirectory);
string installScript = Path.Combine(installationDirectory, _installScriptName);
if (FileSystemHelpers.FileExists(installScript))
{
using (tracer.Step("Execute install.cmd"))
{
OperationManager.Attempt(() =>
{
Executable exe = externalCommandFactory.BuildCommandExecutable(installScript,
installationDirectory,
_settings.GetCommandIdleTimeout(), NullLogger.Instance);
exe.ExecuteWithProgressWriter(NullLogger.Instance, _traceFactory.GetTracer(), installationArgs ?? String.Empty);
});
}
}
}
}
catch (Exception ex)
{
tracer.TraceError(ex);
FileSystemHelpers.DeleteDirectorySafe(installationDirectory);
throw;
}
return(await _localRepository.GetLatestPackageByIdFromSrcRepo(package.Identity.Id));
}
public void DeleteFunction(string name, bool ignoreErrors)
{
FileSystemHelpers.DeleteDirectorySafe(GetFuncPathAndCheckExistence(name), ignoreErrors);
DeleteFunctionArtifacts(name);
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory, _analytics);
return;
}
_inPlaceWorkingDirectory = null;
Dictionary <string, FileInfoBase> sourceDirectoryFileMap = GetJobDirectoryFileMap(JobBinariesPath);
if (WorkingDirectory != null)
{
try
{
var workingDirectoryFileMap = GetJobDirectoryFileMap(WorkingDirectory);
if (!JobDirectoryHasChanged(sourceDirectoryFileMap, workingDirectoryFileMap, _cachedSourceDirectoryFileMap, logger))
{
// no changes detected, so skip the cache/copy step below
return;
}
}
catch (Exception ex)
{
// Log error and ignore it, since this diff optimization isn't critical.
// We'll just do a full copy in this case.
logger.LogWarning("Failed to diff WebJob directories for changes. Continuing to copy WebJob binaries (this will not affect the WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath, _analytics);
_workingDirectory = tempJobInstancePath;
// cache the file map snapshot for next time (to aid in detecting
// file deletions)
_cachedSourceDirectoryFileMap = sourceDirectoryFileMap;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}
public void Delete(int deleteWebRoot = 0, int ignoreErrors = 0)
{
try
{
// Fail if a deployment is in progress
_deploymentLock.LockOperation(() =>
{
using (_tracer.Step("Deleting repository"))
{
string repositoryPath = Path.Combine(_environment.SiteRootPath, Constants.RepositoryPath);
if (String.Equals(repositoryPath, _environment.RepositoryPath, StringComparison.OrdinalIgnoreCase))
{
// Delete the repository
FileSystemHelpers.DeleteDirectorySafe(_environment.RepositoryPath, ignoreErrors != 0);
}
else
{
// Just delete .git folder
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(_environment.RepositoryPath, ".git"), ignoreErrors != 0);
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(_environment.RepositoryPath, ".hg"), ignoreErrors != 0);
}
}
using (_tracer.Step("Delete auto swap lock file"))
{
FileSystemHelpers.DeleteFileSafe(Path.Combine(_environment.LocksPath, AutoSwapHandler.AutoSwapLockFile));
}
using (_tracer.Step("Deleting ssh key"))
{
// Delete the ssh key
FileSystemHelpers.DeleteDirectorySafe(_environment.SSHKeyPath, ignoreErrors != 0);
}
if (deleteWebRoot != 0)
{
// This logic is primarily used to help with site reuse during test.
// The flag is not documented for general use.
using (_tracer.Step("Deleting web root"))
{
// Delete the wwwroot folder
FileSystemHelpers.DeleteDirectoryContentsSafe(_environment.WebRootPath, ignoreErrors != 0);
}
using (_tracer.Step("Deleting diagnostics"))
{
// Delete the diagnostic log. This is a slight abuse of deleteWebRoot, but the
// real semantic is more to reset the site to a fully clean state
FileSystemHelpers.DeleteDirectorySafe(_environment.DiagnosticsPath, ignoreErrors != 0);
}
using (_tracer.Step("Deleting ASP.NET 5 approot"))
{
// Delete the approot folder used by ASP.NET 5 apps
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(_environment.SiteRootPath, "approot"), ignoreErrors != 0);
}
// Delete first deployment manifest since it is no longer needed
FileSystemHelpers.DeleteFileSafe(Path.Combine(_environment.SiteRootPath, Constants.FirstDeploymentManifestFileName));
}
else
{
using (_tracer.Step("Updating initial deployment manifest"))
{
// The active deployment manifest becomes the baseline initial deployment manifest
// When SCM is reconnected, the new deployment will use this manifest to clean the wwwroot
SaveInitialDeploymentManifest();
}
}
using (_tracer.Step("Deleting deployment cache"))
{
// Delete the deployment cache
FileSystemHelpers.DeleteDirectorySafe(_environment.DeploymentsPath, ignoreErrors != 0);
}
}, "Deleting repository", TimeSpan.Zero);
}
catch (LockOperationException ex)
{
HttpResponseMessage response = Request.CreateErrorResponse(HttpStatusCode.Conflict, ex.Message);
throw new HttpResponseException(response);
}
}
/// <summary>
/// This method gets called by the runtime. It is used to add services
/// to the container. It uses the Extension pattern.
/// </summary>
/// <todo>
/// CORE TODO Remove initializing contextAccessor : See if over time we can refactor away the need for this?
/// It's kind of a quick hack/compatibility shim. Ideally you want to get the request context only from where
/// it's specifically provided to you (Request.HttpContext in a controller, or as an Invoke() parameter in
/// a middleware) and pass it wherever its needed.
/// </todo>
public void ConfigureServices(IServiceCollection services)
{
Console.WriteLine(@"Configure Services : " + DateTime.Now.ToString("hh.mm.ss.ffffff"));
FileSystemHelpers.DeleteDirectorySafe("/home/site/locks/deployment");
services.Configure <FormOptions>(options =>
{
options.MultipartBodyLengthLimit = 52428800;
options.ValueCountLimit = 500000;
options.KeyLengthLimit = 500000;
});
services.AddRouteAnalyzer();
// Kudu.Services contains all the Controllers
var kuduServicesAssembly = Assembly.Load("Kudu.Services");
services.AddMvcCore()
.AddRazorPages()
.AddAuthorization()
.AddJsonFormatters()
.AddJsonOptions(options => options.SerializerSettings.ContractResolver = new DefaultContractResolver())
.AddApplicationPart(kuduServicesAssembly).AddControllersAsServices()
.AddApiExplorer();
services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", new Info {
Title = "Kudu API Docs"
});
// Setting the comments path for the Swagger JSON and UI.
var xmlFile = $"Kudu.Services.xml";
var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile);
c.IncludeXmlComments(xmlPath);
});
services.AddGZipCompression();
services.AddDirectoryBrowser();
services.AddDataProtection();
services.AddLogging(
builder =>
{
builder.AddFilter("Microsoft", LogLevel.Warning)
.AddFilter("System", LogLevel.Warning)
.AddConsole();
});
services.AddSingleton <IHttpContextAccessor>(new HttpContextAccessor());
services.TryAddSingleton <ISystemEnvironment>(SystemEnvironment.Instance);
services.AddSingleton <ILinuxConsumptionEnvironment, LinuxConsumptionEnvironment>();
services.AddSingleton <ILinuxConsumptionInstanceManager, LinuxConsumptionInstanceManager>();
services.AddSingleton <IFileSystemPathProvider, FileSystemPathProvider>();
services.AddSingleton <IStorageClient, StorageClient>();
KuduWebUtil.EnsureHomeEnvironmentVariable();
KuduWebUtil.EnsureSiteBitnessEnvironmentVariable();
var fileSystemPathProvider = new FileSystemPathProvider(new MeshPersistentFileSystem(SystemEnvironment.Instance,
new MeshServiceClient(SystemEnvironment.Instance, new HttpClient()), new StorageClient(SystemEnvironment.Instance)));
IEnvironment environment = KuduWebUtil.GetEnvironment(_hostingEnvironment, fileSystemPathProvider);
_webAppRuntimeEnvironment = environment;
services.AddSingleton(_ => new HttpClient());
services.AddSingleton <IMeshServiceClient>(s =>
{
if (environment.IsOnLinuxConsumption)
{
var httpClient = s.GetService <HttpClient>();
var systemEnvironment = s.GetService <ISystemEnvironment>();
return(new MeshServiceClient(systemEnvironment, httpClient));
}
else
{
return(new NullMeshServiceClient());
}
});
services.AddSingleton <IMeshPersistentFileSystem>(s =>
{
if (environment.IsOnLinuxConsumption)
{
var meshServiceClient = s.GetService <IMeshServiceClient>();
var storageClient = s.GetService <IStorageClient>();
var systemEnvironment = s.GetService <ISystemEnvironment>();
return(new MeshPersistentFileSystem(systemEnvironment, meshServiceClient, storageClient));
}
else
{
return(new NullMeshPersistentFileSystem());
}
});
KuduWebUtil.EnsureDotNetCoreEnvironmentVariable(environment);
// CORE TODO Check this
// fix up invalid /home/site/deployments/settings.xml
KuduWebUtil.EnsureValidDeploymentXmlSettings(environment);
// Add various folders that never change to the process path. All child processes will inherit this
KuduWebUtil.PrependFoldersToPath(environment);
// Add middleware for Linux Consumption authentication and authorization
// when KuduLIte is running in service fabric mesh
services.AddLinuxConsumptionAuthentication();
services.AddLinuxConsumptionAuthorization(environment);
// General
services.AddSingleton <IServerConfiguration, ServerConfiguration>();
// CORE TODO Looks like this doesn't ever actually do anything, can refactor out?
services.AddSingleton <IBuildPropertyProvider>(new BuildPropertyProvider());
_noContextDeploymentsSettingsManager =
new DeploymentSettingsManager(new XmlSettings.Settings(KuduWebUtil.GetSettingsPath(environment)));
TraceServices.TraceLevel = _noContextDeploymentsSettingsManager.GetTraceLevel();
// Per request environment
services.AddScoped(sp =>
KuduWebUtil.GetEnvironment(_hostingEnvironment, sp.GetRequiredService <IFileSystemPathProvider>(), sp.GetRequiredService <IDeploymentSettingsManager>()));
services.AddDeploymentServices(environment);
/*
* CORE TODO Refactor ITracerFactory/ITracer/GetTracer()/
* ILogger needs serious refactoring:
* - Names should be changed to make it clearer that ILogger is for deployment
* logging and ITracer and friends are for Kudu tracing
* - ILogger is a first-class citizen in .NET core and has it's own meaning. We should be using it
* where appropriate (and not name-colliding with it)
* - ITracer vs. ITraceFactory is redundant and confusing.
* - TraceServices only serves to confuse stuff now that we're avoiding
*/
Func <IServiceProvider, ITracer> resolveTracer = KuduWebUtil.GetTracer;
ITracer CreateTracerThunk() => resolveTracer(services.BuildServiceProvider());
// First try to use the current request profiler if any, otherwise create a new one
var traceFactory = new TracerFactory(() =>
{
var sp = services.BuildServiceProvider();
var context = sp.GetRequiredService <IHttpContextAccessor>().HttpContext;
return(TraceServices.GetRequestTracer(context) ?? resolveTracer(sp));
});
services.AddScoped <ITracer>(sp =>
{
var context = sp.GetRequiredService <IHttpContextAccessor>().HttpContext;
return(TraceServices.GetRequestTracer(context) ?? NullTracer.Instance);
});
services.AddSingleton <ITraceFactory>(traceFactory);
TraceServices.SetTraceFactory(CreateTracerThunk);
services.AddSingleton <IDictionary <string, IOperationLock> >(
KuduWebUtil.GetNamedLocks(traceFactory, environment));
// CORE TODO ShutdownDetector, used by LogStreamManager.
//var shutdownDetector = new ShutdownDetector();
//shutdownDetector.Initialize()
var noContextTraceFactory = new TracerFactory(() =>
KuduWebUtil.GetTracerWithoutContext(environment, _noContextDeploymentsSettingsManager));
services.AddTransient <IAnalytics>(sp => new Analytics(sp.GetRequiredService <IDeploymentSettingsManager>(),
sp.GetRequiredService <IServerConfiguration>(),
noContextTraceFactory));
// CORE TODO
// Trace shutdown event
// Cannot use shutdownDetector.Token.Register because of race condition
// with NinjectServices.Stop via WebActivator.ApplicationShutdownMethodAttribute
// Shutdown += () => TraceShutdown(environment, noContextDeploymentsSettingsManager);
// LogStream service
services.AddLogStreamService(_webAppRuntimeEnvironment, traceFactory);
// Deployment Service
services.AddWebJobsDependencies();
services.AddScoped <ILogger>(KuduWebUtil.GetDeploymentLogger);
services.AddScoped <IDeploymentManager, DeploymentManager>();
services.AddScoped <IFetchDeploymentManager, FetchDeploymentManager>();
services.AddScoped <IScanManager, ScanManager>();
services.AddScoped <ISSHKeyManager, SSHKeyManager>();
services.AddScoped <IRepositoryFactory>(
sp => KuduWebUtil.GetDeploymentLock(traceFactory, environment).RepositoryFactory =
new RepositoryFactory(
sp.GetRequiredService <IEnvironment>(), sp.GetRequiredService <IDeploymentSettingsManager>(),
sp.GetRequiredService <ITraceFactory>()));
services.AddScoped <IApplicationLogsReader, ApplicationLogsReader>();
// Git server
services.AddGitServer(KuduWebUtil.GetDeploymentLock(traceFactory, environment));
// Git Servicehook Parsers
services.AddGitServiceHookParsers();
services.AddScoped <ICommandExecutor, CommandExecutor>();
// KuduWebUtil.MigrateSite(environment, noContextDeploymentsSettingsManager);
// RemoveOldTracePath(environment);
// RemoveTempFileFromUserDrive(environment);
// CORE TODO Windows Fix: Temporary fix for https://github.com/npm/npm/issues/5905
//EnsureNpmGlobalDirectory();
//EnsureUserProfileDirectory();
//// Skip SSL Certificate Validate
//if (Environment.SkipSslValidation)
//{
// ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
//}
//// Make sure webpages:Enabled is true. Even though we set it in web.config, it could be overwritten by
//// an Azure AppSetting that's supposed to be for the site only but incidently affects Kudu as well.
ConfigurationManager.AppSettings["webpages:Enabled"] = "true";
//// Kudu does not rely owin:appStartup. This is to avoid Azure AppSetting if set.
if (ConfigurationManager.AppSettings?["owin:appStartup"] != null)
{
// Set the appSetting to null since we cannot use AppSettings.Remove(key) (ReadOnly exception!)
ConfigurationManager.AppSettings["owin:appStartup"] = null;
}
//RegisterRoutes(kernel, RouteTable.Routes);
//// Register the default hubs route: ~/signalr
//GlobalHost.DependencyResolver = new SignalRNinjectDependencyResolver(kernel);
//GlobalConfiguration.Configuration.Filters.Add(
// new TraceDeprecatedActionAttribute(
// kernel.Get<IAnalytics>(),
// kernel.Get<ITraceFactory>()));
//GlobalConfiguration.Configuration.Filters.Add(new EnsureRequestIdHandlerAttribute());
//FileTarget target = LogManager.Configuration.FindTargetByName("file") as FileTarget;
//String logfile = _webAppRuntimeEnvironment.LogFilesPath + "/.txt";
//target.FileName = logfile;
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory);
return;
}
_inPlaceWorkingDirectory = null;
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogWarning("Failed to calculate hash for WebJob, continue to copy WebJob binaries (this will not affect WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
_workingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}
// remove old LogFiles/Git trace
private static void RemoveOldTracePath(IEnvironment environment)
{
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(environment.LogFilesPath, "Git"), ignoreErrors: true);
}
private static void RemoveTempFileFromUserDrive(IEnvironment environment)
{
FileSystemHelpers.DeleteDirectorySafe(Path.Combine(environment.RootPath, "data", "Temp"), ignoreErrors: true);
}
private static string CreateCachedRepo(string source, string commitId, IDictionary <string, string> environments)
{
string cachedPath = null;
if (source.IndexOf("github.com", StringComparison.OrdinalIgnoreCase) != -1)
{
// If we're allowed to cache the repository, check if it already exists. If not clone it.
string repoName = Path.GetFileNameWithoutExtension(source.Split('/').Last());
// repo cached per test class to support test parallel run
var context = TestContext.Current;
if (context != null)
{
repoName = GetRepoNamePerContext(context, repoName);
}
cachedPath = Path.Combine(PathHelper.RepositoryCachePath, repoName);
// Check for the actually .git folder, in case some bogus parent exists but is not an actual repo
bool alreadyExists = Directory.Exists(Path.Combine(cachedPath, ".git")) && IsGitRepo(cachedPath);
Executable gitExe = GetGitExe(cachedPath, environments);
if (alreadyExists)
{
TestTracer.Trace("Using cached copy at location {0}", cachedPath);
// Get it into a clean state on the correct commit id
try
{
// If we don't have a specific commit id to go to, use origin/master
if (commitId == null)
{
commitId = "origin/master";
}
GitExecute(gitExe, "reset --hard " + commitId);
}
catch (Exception e)
{
// Some repos like Drupal don't use a master branch (e.g. default branch is 7.x). In those cases,
// simply reset to the HEAD. That won't undo any test commits, but at least it does some cleanup.
if (e.Message.Contains("ambiguous argument 'origin/master'"))
{
GitExecute(gitExe, "reset --hard HEAD");
}
else if (e.Message.Contains("unknown revision"))
{
// If that commit id doesn't exist, try fetching and doing the reset again
// The reason we don't initially fetch is to avoid the network hit when not necessary
GitExecute(gitExe, "fetch origin");
GitExecute(gitExe, "reset --hard " + commitId);
}
else
{
throw;
}
}
GitExecute(gitExe, "clean -dxf");
}
else
{
// Delete any leftover, ignoring errors
FileSystemHelpers.DeleteDirectorySafe(cachedPath);
TestTracer.Trace("Could not find a cached copy at {0}. Cloning from source {1}.", cachedPath, source);
PathHelper.EnsureDirectory(cachedPath);
GitExecute(gitExe, "clone \"{0}\" .", source);
// If we have a commit id, reset to it in case it's older than the latest on our clone
if (commitId != null)
{
GitExecute(gitExe, "reset --hard " + commitId);
}
}
}
return(cachedPath);
}
