public static string ReadAllText(this FileAbstractLayer fal, RelativePath file)
{
using (var sr = OpenReadText(fal, file))
{
return sr.ReadToEnd();
}
}
private static void FixLink(XAttribute link, RelativePath filePath, HashSet<string> linkToFiles)
{
string linkFile;
string anchor = null;
if (PathUtility.IsRelativePath(link.Value))
{
var index = link.Value.IndexOf('#');
if (index == -1)
{
linkFile = link.Value;
}
else if (index == 0)
{
return;
}
else
{
linkFile = link.Value.Remove(index);
anchor = link.Value.Substring(index);
}
var path = filePath + (RelativePath)linkFile;
var file = (string)path.GetPathFromWorkingFolder();
link.Value = file + anchor;
linkToFiles.Add(HttpUtility.UrlDecode(file));
}
}
public static void WriteAllText(this FileAbstractLayer fal, RelativePath file, string content)
{
using (var writer = CreateText(fal, file))
{
writer.Write(content);
}
}
public PathChangeKind GetPathChangeKind(RelativePath path) {
PathChangeKind result;
if (!_map.TryGetValue(path, out result)) {
result = PathChangeKind.None;
}
return result;
}
public static string GetProperty(this FileAbstractLayer fal, RelativePath file, string propertyName)
{
var dict = fal.GetProperties(file);
string result;
dict.TryGetValue(propertyName, out result);
return result;
}
public IEnumerable<string> GetSection(string sectionName, Func<IEnumerable<string>, IEnumerable<string>> postProcessing)
{
var filename = new RelativePath(sectionName);
return _configurationFileProvider.ReadFile(filename, (fullPathName, lines) => {
_volatileToken.AddFile(fullPathName);
return postProcessing(lines);
});
}
public override void Copy(PathMapping sourceFilePath, RelativePath destFilePath)
{
var key = destFilePath.GetPathFromWorkingFolder();
_mapping[key] = new PathMapping(key, sourceFilePath.PhysicalPath)
{
Properties = sourceFilePath.Properties,
};
}
public FileName(DirectoryName parent, RelativePath relativePath) {
if (parent == null)
throw new ArgumentNullException("parent");
if (relativePath.IsEmpty)
throw new ArgumentException("Relative path is empty", "relativePath");
_parent = parent;
_relativePath = relativePath;
}
public void RelativePath_Unit_Append1_PathContainsAnotherSeparator()
{
String[] nodes = new String[] { "sites" };
Char separator = RelativePath.ForwardSlash;
RelativePath target = new RelativePath(nodes, separator);
String path = "Chad\\Greer";
target.Append(path);
}
public PhpSourceFile(FullPath root, FullPath fullPath)
{
root.EnsureNonEmpty("root");
fullPath.EnsureNonEmpty("fullPath");
this.fullPath = fullPath;
this.relativePath = RelativePath.Empty;
this.root = root;
}
public PathMapping? FindFile(RelativePath file)
{
var pp = Path.Combine(_expandedInputFolder, file.RemoveWorkingFolder());
if (!File.Exists(pp))
{
return null;
}
return new PathMapping(file, Path.Combine(InputFolder, file.RemoveWorkingFolder())) { Properties = Properties };
}
public bool Exists(RelativePath file)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
EnsureNotDisposed();
return Reader.FindFile(file) != null;
}
public FileStream OpenRead(RelativePath file)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
EnsureNotDisposed();
var pp = FindPhysicalPath(file);
return File.OpenRead(Environment.ExpandEnvironmentVariables(pp.PhysicalPath));
}
public bool MatchFileName(RelativePath relativePath, IPathComparer comparer) {
var path = relativePath.Value;
CheckPath(path);
if (PrePassWontMatch(MatchKind.File, path, comparer))
return false;
var result = BaseOperator.Match(MatchKind.File, comparer, Operators, 0, path, 0);
return IsMatch(path, result);
}
public IEnumerable<string> ReadFile(RelativePath relativePath, Func<FullPath, IEnumerable<string>, IEnumerable<string>> postProcessing)
{
foreach (var directoryName in PossibleDirectoryNames()) {
var path = directoryName.Combine(relativePath);
if (_fileSystem.FileExists(path))
return postProcessing(path, _fileSystem.ReadAllLines(path));
}
throw new FileLoadException(
string.Format("Could not load configuration file \"{0}\" from the following locations:{1}", relativePath,
PossibleDirectoryNames().Aggregate("", (x, y) => x + "\n" + y)));
}
public void RelativePath_Unit_Append1_Optimal()
{
String[] nodes = new String[] { "sites" };
Char separator = RelativePath.ForwardSlash;
RelativePath target = new RelativePath(nodes, separator);
String path = "Chad/Greer";
RelativePath result = target.Append(path);
Assert.AreEqual("sites", result.Nodes.ElementAt(0));
Assert.AreEqual("Chad", result.Nodes.ElementAt(1));
Assert.AreEqual("Greer", result.Nodes.ElementAt(2));
Assert.AreEqual(separator, result.Separator);
}
public FileStream Create(RelativePath file)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
EnsureNotDisposed();
if (!CanWrite)
{
throw new InvalidOperationException();
}
return Writer.Create(file);
}
public void TestPaths()
{
FullPath root;
FullPath full;
RelativePath rel;
string str1, str2;
string[,] cases =
{
// root: // full: // full canonical: // relative canonical:
{ @"C:\a/b/c/", @"D:\a/b/", @"D:\a\b\", @"D:\a\b\" },
{ @"C:\a\b\c", @"C:\a\b\c", @"C:\a\b\c", @"" },
{ @"C:\a\b\c", @"C:\a\b\c\", @"C:\a\b\c\", @"" },
{ @"C:\a\b\c\", @"C:\a\b\c", @"C:\a\b\c", @"" },
{ @"C:\a\b\c\", @"C:\a\b\c\", @"C:\a\b\c\", @"" },
{ @"C:\a\b\c", @"C:\a\b", @"C:\a\b", @".." },
{ @"C:\a\b\c\", @"C:\a\b", @"C:\a\b", @".." },
{ @"C:\a\b\c", @"C:\", @"C:\", @"..\..\.." },
{ @"C:\a\b\c\", @"C:\", @"C:\", @"..\..\.." },
{ @"C:\a\b\c\", @"C:\a\b\x\y\z", @"C:\a\b\x\y\z", @"..\x\y\z" },
{ @"C:\a\b\cd\", @"C:\a\b\c", @"C:\a\b\c", @"..\c" },
{ @"C:\a\b\cd", @"C:\a\b\c", @"C:\a\b\c", @"..\c" },
{ @"C:\a\b\cd\", @"C:\a\b\c\d", @"C:\a\b\c\d", @"..\c\d" },
{ @"C:\a\b\cd", @"C:\a\b\c\d", @"C:\a\b\c\d", @"..\c\d" },
};
for (int i = 0; i < cases.GetLength(0); i++)
{
root = new FullPath(cases[i, 0]);
full = new FullPath(cases[i, 1]);
rel = new RelativePath(root, full);
Assert.AreEqual(full.ToString(), cases[i, 2]);
Assert.AreEqual(rel.ToString(), cases[i, 3]);
str1 = full;
if (str1[str1.Length - 1] == '\\') str1 = str1.Substring(0, str1.Length - 1);
str2 = rel.ToFullPath(root);
if (str2[str2.Length - 1] == '\\') str2 = str2.Substring(0, str2.Length - 1);
Assert.AreEqual(str1, str2);
Assert.AreEqual(RelativePath.ParseCanonical(cases[i, 3]).ToString(), cases[i, 3]);
}
}
public bool MatchFileName(RelativePath relativePath, IPathComparer comparer) {
if (relativePath.IsEmpty)
throw new ArgumentNullException("relativePath");
if (_fileExtensions.Contains(relativePath.Extension))
return true;
// Note: Use "for" loop to avoid allocation if using "Any()"
for (var index = 0; index < _pathMatchers.Length; index++) {
if (_pathMatchers[index].MatchFileName(relativePath, comparer))
return true;
}
return false;
}
public PathMapping(RelativePath logicalPath, string physicalPath)
{
if (logicalPath == null)
{
throw new ArgumentNullException(nameof(logicalPath));
}
if (physicalPath == null)
{
throw new ArgumentNullException(nameof(physicalPath));
}
LogicalPath = logicalPath.GetPathFromWorkingFolder();
PhysicalPath = physicalPath;
AllowMoveOut = false;
Properties = ImmutableDictionary<string, string>.Empty;
}
public override FileStream Create(RelativePath file)
{
var key = file.GetPathFromWorkingFolder();
PathMapping pm;
if (_mapping.TryGetValue(key, out pm) &&
pm.PhysicalPath.StartsWith(OutputFolder))
{
return File.Create(Environment.ExpandEnvironmentVariables(pm.PhysicalPath));
}
var pair = CreateRandomFileStream();
_mapping[key] =
new PathMapping(
key,
Path.Combine(OutputFolder, pair.Item1));
return pair.Item2;
}
public void Copy(RelativePath sourceFileName, RelativePath destFileName)
{
if (sourceFileName == null)
{
throw new ArgumentNullException(nameof(sourceFileName));
}
if (destFileName == null)
{
throw new ArgumentNullException(nameof(destFileName));
}
EnsureNotDisposed();
if (!CanWrite)
{
throw new InvalidOperationException();
}
var mapping = FindPhysicalPath(sourceFileName);
Writer.Copy(mapping, destFileName);
}
private void GetRootTocFromOutputRoot(SystemMetadata attrs, RelativePath file)
{
var rootToc = _toc.FirstOrDefault();
if (rootToc != null)
{
var rootTocPath = rootToc.File.RemoveWorkingFolder();
if (rootTocPath.SubdirectoryCount == 0)
{
attrs.RootTocPath = rootTocPath;
var rootTocRelativePath = rootTocPath.MakeRelativeTo(file);
attrs.RelativePathToRootToc = rootTocRelativePath;
attrs.RootTocKey = rootToc.Key;
Logger.LogVerbose($"Root TOC file {rootTocPath} is found.");
}
else
{
Logger.LogVerbose(
$"Root TOC file from output folder is not found, the toppest TOC file is {rootTocPath}");
}
}
}
public PathMapping? FindFile(RelativePath file)
{
var path = file.GetPathFromWorkingFolder();
foreach (var m in Mappings)
{
if (m.IsFolder)
{
var localPath = path - m.LogicalPath;
if (m.AllowMoveOut || localPath.ParentDirectoryCount == 0)
{
var physicalPath = Path.Combine(m.PhysicalPath, localPath.ToString());
if (File.Exists(Environment.ExpandEnvironmentVariables(physicalPath)))
{
return new PathMapping(path, physicalPath) { Properties = m.Properties };
}
}
}
else if (m.LogicalPath == path)
{
return m;
}
}
return null;
}
public static bool HasProperty(this FileAbstractLayer fal, RelativePath file, string propertyName)
{
var dict = fal.GetProperties(file);
return dict.ContainsKey(propertyName);
}
public static StreamWriter CreateText(this FileAbstractLayer fal, RelativePath file) =>
new StreamWriter(fal.Create(file));
public static StreamReader OpenReadText(this FileAbstractLayer fal, RelativePath file) =>
new StreamReader(fal.OpenRead(file));
public void ContainerPathIsContainer()
{
var path = RelativePath.CreateFromSegments("a", "b", "c").AsContainer();
Assert.That(path.IsContainer(), Is.True);
}
/// <summary>
/// Creates an output file artifact.
/// </summary>
public FileArtifact CreateOutputFile(string relative) =>
FileArtifact.CreateOutputFile(m_objectRoot.Combine(
Context.PathTable, RelativePath.Create(Context.StringTable, relative)));
public static ProjectFile ToProjectFile(ProjectModel project)
{
ProjectFile file = new ProjectFile();
file.Stub.Type = project.Stub.Type;
file.Stub.IconPath = ConvertAbsoluteToRelativePath(project.Stub.IconPath.ToNullIfEmpty());
file.Stub.Padding = project.Stub.Padding;
file.Startup.Melt = project.Startup.Melt;
file.VersionInfo.FileDescription = project.VersionInfo.FileDescription.ToNullIfEmptyOrWhiteSpace();
file.VersionInfo.ProductName = project.VersionInfo.ProductName.ToNullIfEmptyOrWhiteSpace();
file.VersionInfo.FileVersion = project.VersionInfo.FileVersion1 != 0 || project.VersionInfo.FileVersion2 != 0 || project.VersionInfo.FileVersion3 != 0 || project.VersionInfo.FileVersion4 != 0 ? project.VersionInfo.FileVersion1 + "." + project.VersionInfo.FileVersion2 + "." + project.VersionInfo.FileVersion3 + "." + project.VersionInfo.FileVersion4 : null;
file.VersionInfo.ProductVersion = project.VersionInfo.ProductVersion.ToNullIfEmptyOrWhiteSpace();
file.VersionInfo.Copyright = project.VersionInfo.Copyright.ToNullIfEmptyOrWhiteSpace();
file.VersionInfo.OriginalFilename = project.VersionInfo.OriginalFilename.ToNullIfEmptyOrWhiteSpace();
if (project.Manifest.UseTemplate)
{
file.Manifest.Template = project.Manifest.Template;
}
else if (project.Manifest.UseFile)
{
file.Manifest.Path = ConvertAbsoluteToRelativePath(project.Manifest.Path.ToNullIfEmpty());
}
foreach (ProjectItemModel item in project.Items)
{
ProjectSource source;
if (item is ProjectMessageBoxItemModel)
{
source = null;
}
else
{
switch (item.Source)
{
case ProjectItemSource.Embedded:
file.AddSource(source = new EmbeddedSource
{
Path = ConvertAbsoluteToRelativePath(item.SourceEmbeddedPath.ToNullIfEmpty()),
Compress = item.SourceEmbeddedCompress,
EofData = item.SourceEmbeddedEofData
});
break;
case ProjectItemSource.Download:
file.AddSource(source = new DownloadSource
{
Url = item.SourceDownloadUrl
});
break;
default:
throw new InvalidEnumArgumentException();
}
source.Id = item.SourceId;
}
if (item is ProjectRunPEItemModel)
{
file.AddAction(new RunPEAction
{
Source = source
});
}
else if (item is ProjectInvokeItemModel)
{
file.AddAction(new InvokeAction
{
Source = source
});
}
else if (item is ProjectDropItemModel dropItem)
{
file.AddAction(new DropAction
{
Source = source,
Location = dropItem.Location,
FileName = dropItem.FileName,
FileAttributeHidden = dropItem.FileAttributeHidden,
FileAttributeSystem = dropItem.FileAttributeSystem,
ExecuteVerb = dropItem.ExecuteVerb
});
}
else if (item is ProjectMessageBoxItemModel messageBoxItem)
{
file.AddAction(new MessageBoxAction
{
Title = messageBoxItem.Title,
Text = messageBoxItem.Text,
Icon = messageBoxItem.Icon,
Buttons = messageBoxItem.Buttons,
OnOk = MessageBoxAction.HasEvent(messageBoxItem.Buttons, MessageBoxEvent.Ok) ? messageBoxItem.OnOk : ActionEvent.None,
OnCancel = MessageBoxAction.HasEvent(messageBoxItem.Buttons, MessageBoxEvent.Cancel) ? messageBoxItem.OnCancel : ActionEvent.None,
OnYes = MessageBoxAction.HasEvent(messageBoxItem.Buttons, MessageBoxEvent.Yes) ? messageBoxItem.OnYes : ActionEvent.None,
OnNo = MessageBoxAction.HasEvent(messageBoxItem.Buttons, MessageBoxEvent.No) ? messageBoxItem.OnNo : ActionEvent.None,
OnAbort = MessageBoxAction.HasEvent(messageBoxItem.Buttons, MessageBoxEvent.Abort) ? messageBoxItem.OnAbort : ActionEvent.None,
OnRetry = MessageBoxAction.HasEvent(messageBoxItem.Buttons, MessageBoxEvent.Retry) ? messageBoxItem.OnRetry : ActionEvent.None,
OnIgnore = MessageBoxAction.HasEvent(messageBoxItem.Buttons, MessageBoxEvent.Ignore) ? messageBoxItem.OnIgnore : ActionEvent.None
});
}
else
{
throw new InvalidOperationException();
}
}
return(file);
string ConvertAbsoluteToRelativePath(string absolutePath)
{
return(project.ProjectPath == null ? absolutePath : RelativePath.AbsoluteToRelativePath(Path.GetDirectoryName(project.ProjectPath), absolutePath));
}
}
public override async ValueTask <Directive?> Run(RawSourceFile source)
{
if (_isGenericGame)
{
if (source.Path.StartsWith(Consts.GameFolderFilesDir))
{
return(null);
}
}
var name = source.File.Name.FileName;
RelativePath nameWithoutExt = name;
if (name.Extension == Consts.MOHIDDEN)
{
nameWithoutExt = name.FileNameWithoutExtension;
}
if (!_indexed.TryGetValue(name, out var choices))
{
_indexed.TryGetValue(nameWithoutExt, out choices);
}
dynamic?modIni = null;
if (_compiler is MO2Compiler)
{
if (_bsa == null && source.File.IsNative && source.AbsolutePath.InFolder(((MO2Compiler)_compiler).MO2ModsFolder))
{
((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(source.AbsolutePath), out modIni);
}
else if (_bsa != null)
{
var bsaPath = _bsa.FullPath.Base;
((MO2Compiler)_compiler).ModInis.TryGetValue(ModForFile(bsaPath), out modIni);
}
}
var installationFile = (string?)modIni?.General?.installationFile;
VirtualFile[] found = {};
// Find based on exact file name + ext
if (choices != null && installationFile != null)
{
var relName = (RelativePath)Path.GetFileName(installationFile);
found = choices.Where(f => f.FilesInFullPath.First().Name.FileName == relName).ToArray();
}
// Find based on file name only (not ext)
if (found.Length == 0 && choices != null)
{
found = choices.ToArray();
}
// Find based on matchAll=<archivename> in [General] in meta.ini
var matchAllName = (string?)modIni?.General?.matchAll;
if (matchAllName != null && found.Length == 0)
{
var relName = (RelativePath)Path.GetFileName(matchAllName);
if (_indexedByName.TryGetValue(relName, out var arch))
{
var dist = new Levenshtein();
found = arch.SelectMany(a => a.ThisAndAllChildren)
.OrderBy(a => dist.Distance(a.Name.FileName.ToString(), source.File.Name.FileName.ToString()))
.Take(3)
.ToArray();
}
}
if (found.Length == 0)
{
return(null);
}
var e = source.EvolveTo <PatchedFromArchive>();
var patches = found.Select(c => (Utils.TryGetPatch(c.Hash, source.File.Hash, out var data), data, c))
.ToArray();
if (patches.All(p => p.Item1))
{
var(_, bytes, file) = PickPatch(_compiler, patches);
e.FromHash = file.Hash;
e.ArchiveHashPath = file.MakeRelativePaths();
e.PatchID = await _compiler.IncludeFile(bytes !);
}
else
{
e.Choices = found;
}
if (source.File.IsNative && await VirusScanner.ShouldScan(source.File.AbsoluteName))
{
if (await ClientAPI.GetVirusScanResult(source.File.AbsoluteName) == VirusScanner.Result.Malware)
{
Utils.ErrorThrow(new Exception($"Executable file {source.File.AbsoluteName} ({source.File}) has been marked as malware."));
}
}
return(e);
}
public override bool LoadAllGames()
{
SteamUniverses ??= LoadUniverses();
if (SteamUniverses.Count == 0)
{
Utils.Log("Could not find any Steam Libraries");
return(false);
}
SteamUniverses.Do(u =>
{
Utils.Log($"Searching for Steam Games in {u}");
u.EnumerateFiles(false, "*.acf")
.Where(a => a.Exists)
.Where(a => a.IsFile)
.Do(f =>
{
var game = new SteamGame();
var gotID = false;
f.ReadAllLines().Do(l =>
{
if (l.ContainsCaseInsensitive("\"appid\""))
{
if (!int.TryParse(GetVdfValue(l), out var id))
{
return;
}
game.ID = id;
gotID = true;
}
if (l.ContainsCaseInsensitive("\"name\""))
{
game.Name = GetVdfValue(l);
}
if (!l.ContainsCaseInsensitive("\"installdir\""))
{
return;
}
var value = GetVdfValue(l);
AbsolutePath absPath;
if (Path.IsPathRooted(value))
{
absPath = (AbsolutePath)value;
}
else
{
absPath = new RelativePath("common").Combine(GetVdfValue(l)).RelativeTo(u);
}
if (absPath.Exists)
{
game.Path = absPath;
}
});
if (!gotID || !game.Path.IsDirectory)
{
return;
}
var gameMeta = GameRegistry.Games.Values.FirstOrDefault(g => g.SteamIDs?.Contains(game.ID) ?? false);
if (gameMeta == null)
{
Utils.Log($"Steam Game \"{game.Name}\" ({game.ID}) is not supported, skipping");
return;
}
game.Game = gameMeta.Game;
game.Universe = u;
Utils.Log($"Found Steam Game: \"{game.Name}\" ({game.ID}) at {game.Path}");
LoadWorkshopItems(game);
Games.Add(game);
});
});
Utils.Log($"Total number of Steam Games found: {Games.Count}");
return(true);
}
private static void LoadWorkshopItems(SteamGame game)
{
var workshop = new RelativePath("workshop").RelativeTo(game.Universe);
if (!workshop.Exists)
{
return;
}
workshop.EnumerateFiles(false, "*.acf")
.Where(f => f.Exists)
.Where(f => f.IsFile)
.Do(f =>
{
if (f.FileName.ToString() != $"appworkshop{game.ID}.acf")
{
return;
}
Utils.Log($"Found Steam Workshop item file {f} for \"{game.Name}\"");
var lines = f.ReadAllLines().ToList();
var end = false;
var foundAppID = false;
var workshopItemsInstalled = 0;
var workshopItemDetails = 0;
var bracketStart = 0;
var bracketEnd = 0;
SteamWorkshopItem?currentItem = new SteamWorkshopItem(game);
lines.Do(l =>
{
if (end)
{
return;
}
currentItem ??= new SteamWorkshopItem(game);
var currentLine = lines.IndexOf(l);
if (l.ContainsCaseInsensitive("\"appid\"") && !foundAppID)
{
if (!int.TryParse(GetVdfValue(l), out var appID))
{
return;
}
foundAppID = true;
if (appID != game.ID)
{
return;
}
}
if (!foundAppID)
{
return;
}
if (l.ContainsCaseInsensitive("\"SizeOnDisk\""))
{
if (!int.TryParse(GetVdfValue(l), out var sizeOnDisk))
{
return;
}
game.WorkshopItemsSizeOnDisk += sizeOnDisk;
}
if (l.ContainsCaseInsensitive("\"WorkshopItemsInstalled\""))
{
workshopItemsInstalled = currentLine;
}
if (l.ContainsCaseInsensitive("\"WorkshopItemDetails\""))
{
workshopItemDetails = currentLine;
}
if (workshopItemsInstalled == 0)
{
return;
}
if (currentLine <= workshopItemsInstalled + 1 && currentLine >= workshopItemDetails - 1)
{
return;
}
if (currentItem.ItemID == 0)
{
if (!int.TryParse(GetSingleVdfValue(l), out currentItem.ItemID))
{
return;
}
}
if (currentItem.ItemID == 0)
{
return;
}
if (bracketStart == 0 && l.Contains("{"))
{
bracketStart = currentLine;
}
if (bracketEnd == 0 && l.Contains("}"))
{
bracketEnd = currentLine;
}
if (bracketStart == 0)
{
return;
}
if (currentLine == bracketStart + 1)
{
if (!int.TryParse(GetVdfValue(l), out currentItem.Size))
{
return;
}
}
if (bracketStart == 0 || bracketEnd == 0 || currentItem.ItemID == 0 || currentItem.Size == 0)
{
return;
}
bracketStart = 0;
bracketEnd = 0;
game.WorkshopItems.Add(currentItem);
Utils.Log($"Found Steam Workshop item {currentItem.ItemID}");
currentItem = null;
end = true;
});
});
}
/// <summary>
/// Creates a new PipConstructionHelper
/// </summary>
/// <remarks>
/// Ideally this function would take ModuleId, FullSymbol QualifierId and compute uniqueOutputLocation itself. Unfortunately today the data is not yet
/// exposed via IPipGraph, therefore the responsibility is on the call site for now.
/// </remarks>
public static PipConstructionHelper Create(
PipExecutionContext context,
AbsolutePath objectRoot,
AbsolutePath redirectedRoot,
AbsolutePath tempRoot,
IPipGraph pipGraph,
ModuleId moduleId,
string moduleName,
RelativePath specRelativePath,
FullSymbol symbol,
LocationData thunkLocation,
QualifierId qualifierId)
{
var stringTable = context.StringTable;
var pathTable = context.PathTable;
// We have to manually compute the pipPipUniqueString here, Ideally we pass PackageId, SpecFile, FullSymbol and qualiferId and have it computed inside, but the IPipGraph does not allow querying it for now.
string hashString;
long semiStableHashSeed = 0;
using (var builderWrapper = Pools.GetStringBuilder())
{
var builder = builderWrapper.Instance;
builder.Append(moduleName);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.GetOrdinalHashCode64(moduleName);
if (specRelativePath.IsValid)
{
string specPath = specRelativePath.ToString(stringTable);
builder.Append(specPath);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(specPath));
}
var symbolName = symbol.ToStringAsCharArray(context.SymbolTable);
builder.Append(symbolName);
builder.Append('/');
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(symbolName));
var qualifierDisplayValue = context.QualifierTable.GetCanonicalDisplayString(qualifierId);
builder.Append(qualifierDisplayValue);
semiStableHashSeed = HashCodeHelper.Combine(semiStableHashSeed, HashCodeHelper.GetOrdinalHashCode64(qualifierDisplayValue));
var pipPipUniqueString = builder.ToString();
hashString = Hash(pipPipUniqueString);
}
var pipRelativePath = RelativePath.Create(
PathAtom.Create(stringTable, hashString.Substring(0, 1)),
PathAtom.Create(stringTable, hashString.Substring(1, 1)),
PathAtom.Create(stringTable, hashString.Substring(2)));
var valuePip = new ValuePip(symbol, qualifierId, thunkLocation);
return(new PipConstructionHelper(
context,
objectRoot,
redirectedRoot,
tempRoot,
pipGraph,
moduleId,
moduleName,
valuePip,
pipRelativePath,
semiStableHashSeed));
}
public FileSystemInfo GetFullyQualifiedPath(RelativePath path)
{
return(null);
}
public NinjaPipConstructor(FrontEndContext context, FrontEndHost frontEndHost, string frontEndName, ModuleDefinition moduleDefinition, QualifierId qualifierId, AbsolutePath projectRoot, AbsolutePath specPath, bool suppressDebugFlags, IUntrackingSettings untrackingSettings)
{
Contract.Requires(context != null);
Contract.Requires(frontEndHost != null);
Contract.Requires(moduleDefinition != null);
Contract.Requires(projectRoot.IsValid);
Contract.Requires(specPath.IsValid);
m_context = context;
m_frontEndHost = frontEndHost;
m_moduleDefinition = moduleDefinition;
m_projectRoot = projectRoot;
m_specPath = specPath;
m_suppressDebugFlags = suppressDebugFlags;
m_untrackingSettings = untrackingSettings;
m_pipConstructionHelper = GetPipConstructionHelperForModule(m_projectRoot, moduleDefinition, qualifierId);
m_frontEndName = frontEndName;
m_manuallyDroppedDependenciesPath = Lazy.Create(() => m_frontEndHost.Configuration.Layout.BuildEngineDirectory
.Combine(m_context.PathTable, RelativePath.Create(m_context.StringTable, @"tools\CMakeNinjaPipEnvironment")));
// Lazy initialization of environment variables and passthroughs
var allEnvironmentVariables = Lazy.Create(GetAllEnvironmentVariables);
m_environmentVariables = Lazy.Create(() => allEnvironmentVariables.Value.Where(kvp => SpecialEnvironmentVariables.PassThroughPrefixes.All(prefix => !kvp.Key.StartsWith(prefix))));
m_passThroughEnvironmentVariables = Lazy.Create(() => allEnvironmentVariables.Value.Where(kvp => SpecialEnvironmentVariables.PassThroughPrefixes.Any(prefix => kvp.Key.StartsWith(prefix))));
}
public void FilePathIsNotContainer()
{
var path = RelativePath.CreateFromSegments("a", "b", "c");
Assert.That(path.IsContainer(), Is.False);
}
/// <summary>
/// Emits a static inclusion.
/// </summary>
private PhpTypeCode EmitStaticInclusion(IncludingEx node, CodeGenerator /*!*/ codeGenerator)
{
ILEmitter il = codeGenerator.IL;
Label endif_label = il.DefineLabel();
Label else_label = il.DefineLabel();
MethodInfo method;
// if the expression should be emitted:
if (characteristic == Characteristic.StaticArgEvaluated)
{
if (!(node.Target is StringLiteral || node.Target is BinaryStringLiteral))
{
// emits expression evaluation and ignores the result:
node.Target.Emit(codeGenerator);
il.Emit(OpCodes.Pop);
}
}
if (characteristic == Characteristic.StaticAutoInclusion)
{
// calls the Main routine only if this script is the main one:
il.Ldarg(ScriptBuilder.ArgIsMain);
}
else
{
RelativePath relativePath = new RelativePath(inclusion.Includee.RelativeSourcePath); // normalize the relative path
// CALL context.StaticInclude(<relative included script source path>,<this script type>,<inclusion type>);
codeGenerator.EmitLoadScriptContext();
il.Emit(OpCodes.Ldc_I4, (int)relativePath.Level);
il.Emit(OpCodes.Ldstr, relativePath.Path);
il.Emit(OpCodes.Ldtoken, inclusion.Includee.ScriptClassType);
il.LoadLiteral(node.InclusionType);
il.Emit(OpCodes.Call, Methods.ScriptContext.StaticInclude);
}
// IF (STACK)
il.Emit(OpCodes.Brfalse, else_label);
if (true)
{
// emits a call to the main helper of the included script:
method = inclusion.Includee.MainHelper;
// CALL <Main>(context, variables, self, includer, false):
codeGenerator.EmitLoadScriptContext();
codeGenerator.EmitLoadRTVariablesTable();
codeGenerator.EmitLoadSelf();
codeGenerator.EmitLoadClassContext();
il.Emit(OpCodes.Ldc_I4_0);
il.Emit(OpCodes.Call, method);
il.Emit(OpCodes.Br, endif_label);
}
// ELSE
il.MarkLabel(else_label);
if (true)
{
// LOAD <PhpScript.SkippedIncludeReturnValue>;
il.LoadLiteral(ScriptModule.SkippedIncludeReturnValue);
il.Emit(OpCodes.Box, ScriptModule.SkippedIncludeReturnValue.GetType());
}
il.MarkLabel(endif_label);
// END IF
return(PhpTypeCode.Object);
}
public VendorSample BuildVendorSample(string rootDir, string boardName, SpecializedDevice device, string package, HashSet <string> allComponentIDs, HashSet <string> implicitFrameworks)
{
Dictionary <string, string> properties = new Dictionary <string, string>();
if (!string.IsNullOrEmpty(ExplicitFPUSetting))
{
properties["com.sysprogs.bspoptions.arm.floatmode"] = ExplicitFPUSetting.Contains("hard") ? "-mfloat-abi=hard" : "-mfloat-abi=soft";
}
string sampleName = ID;
if (!string.IsNullOrEmpty(RelativePath))
{
string[] relativePathComponents = RelativePath.Split('/');
if (relativePathComponents.Length > 3 && relativePathComponents[0] == "boards")
{
string expectedPrefix = relativePathComponents[1] + "_" + relativePathComponents[2] + "_";
if (sampleName.StartsWith(expectedPrefix))
{
sampleName = sampleName.Substring(expectedPrefix.Length);
}
}
}
VendorSample sample = new VendorSample
{
DeviceID = device.MakeMCUID(package),
UserFriendlyName = sampleName,
InternalUniqueID = ID,
Description = Description,
BoardName = boardName,
Configuration = new VendorSampleConfiguration
{
Frameworks = Dependencies.Where(d => allComponentIDs.Contains(d)).Select(d => ParsedComponent.FrameworkIDPrefix + d).Concat(implicitFrameworks).Distinct().ToArray(),
MCUConfiguration = new PropertyDictionary2(properties)
},
VirtualPath = Category,
PreprocessorMacros = Defines,
NoImplicitCopy = true
};
List <string> sources = new List <string>(), headers = new List <string>();
HashSet <string> includeDirectories = new HashSet <string>();
string[] matchingPathComponents = null;
foreach (var lst in SourceLists)
{
foreach (var file in lst.LocateAllFiles(device, rootDir))
{
var bspPath = file.GetBSPPath();
UpdateMatchingPathComponents(bspPath, ref matchingPathComponents);
file.UpdateIncludeDirectoryList(includeDirectories);
switch (file.Type)
{
case SourceType.Library:
case SourceType.Source:
sources.Add(bspPath);
break;
case SourceType.Header:
headers.Add(bspPath);
break;
case SourceType.LinkerScript:
sample.LinkerScript = bspPath;
break;
}
}
}
sample.CLanguageStandard = MapLanguageStandard(LanguageStandard);
sample.LDFLAGS = string.Join(" ", AdvancedLinkerOptions.Where(f => f.StartsWith("--defsym")).Select(f => "-Wl," + f).ToArray());
if (sample.LDFLAGS == "")
{
sample.LDFLAGS = null;
}
if (matchingPathComponents != null)
{
sample.Path = string.Join("/", matchingPathComponents);
}
sample.SourceFiles = sources.ToArray();
sample.HeaderFiles = headers.ToArray();
sample.IncludeDirectories = includeDirectories.ToArray();
return(sample);
}
private Task <SandboxedProcessResult> ExecuteCMakeRunner(AbsolutePath argumentsFile, IEnumerable <AbsolutePath> searchLocations)
{
string rootString = ProjectRoot.ToString(m_context.PathTable);
AbsolutePath outputDirectory = argumentsFile.GetParent(m_context.PathTable);
FileUtilities.CreateDirectory(outputDirectory.ToString(m_context.PathTable)); // Ensure it exists
SerializeToolArguments(argumentsFile, searchLocations);
void CleanUpOnResult()
{
try
{
FileUtilities.DeleteFile(argumentsFile.ToString(m_context.PathTable));
}
catch (BuildXLException e)
{
Tracing.Logger.Log.CouldNotDeleteToolArgumentsFile(
m_context.LoggingContext,
m_resolverSettings.Location(m_context.PathTable),
argumentsFile.ToString(m_context.PathTable),
e.Message);
}
}
var environment = FrontEndUtilities.GetEngineEnvironment(m_host.Engine, CMakeFrontEnd.Name);
// TODO: This manual configuration is temporary. Remove after the cloud builders have the correct configuration
var pathToManuallyDroppedTools = m_configuration.Layout.BuildEngineDirectory.Combine(m_context.PathTable, RelativePath.Create(m_context.StringTable, @"tools\CmakeNinjaPipEnvironment"));
if (FileUtilities.Exists(pathToManuallyDroppedTools.ToString(m_context.PathTable)))
{
environment = SpecialCloudConfiguration.OverrideEnvironmentForCloud(environment, pathToManuallyDroppedTools, m_context);
}
var buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(new ReadOnlyDictionary <string, string>(environment));
return(FrontEndUtilities.RunSandboxedToolAsync(
m_context,
m_pathToTool.ToString(m_context.PathTable),
buildStorageDirectory: outputDirectory.ToString(m_context.PathTable),
fileAccessManifest: GenerateFileAccessManifest(m_pathToTool.GetParent(m_context.PathTable)),
arguments: I($@"""{argumentsFile.ToString(m_context.PathTable)}"""),
workingDirectory: rootString,
description: "CMakeRunner",
buildParameters,
onResult: CleanUpOnResult));
}
public FileInfo(string key, RelativePath file)
{
Key = key;
File = file;
}
public void CannotCreatePathWithNoSegments()
{
Assert.Throws <ArgumentException>(() => RelativePath.CreateFromSegments());
}
/// <summary>
/// Extracts a file into a folder with in manifest based incrementality.
/// </summary>
internal async Task <EvaluationResult> PerformExtractOrIncrementalCheckAsync(DownloadData downloadData)
{
if (m_context.CancellationToken.IsCancellationRequested)
{
return(EvaluationResult.Canceled);
}
// Ensure file is downloaded
var extractedFileResult = await DownloadFile(downloadData);
if (extractedFileResult.IsErrorValue)
{
return(extractedFileResult);
}
var extractedFile = (FileArtifact)extractedFileResult.Value;
var moduleDescriptor = m_workspaceResolver.GetModuleDescriptor(downloadData);
var pipConstructionHelper = PipConstructionHelper.Create(
m_context,
m_frontEndHost.Engine.Layout.ObjectDirectory,
m_frontEndHost.Engine.Layout.RedirectedDirectory,
m_frontEndHost.Engine.Layout.TempDirectory,
m_frontEndHost.PipGraph,
moduleDescriptor.Id,
moduleDescriptor.Name,
RelativePath.Create(downloadData.ModuleSpecFile.GetName(m_context.PathTable)),
FullSymbol.Create(m_context.SymbolTable, "extracted"),
new LocationData(downloadData.ModuleSpecFile, 0, 0),
m_context.QualifierTable.EmptyQualifierId);
// When we don't have to extract we'll expose the downloaded file in the contents.
if (downloadData.Settings.ArchiveType == DownloadArchiveType.File)
{
return(SealDirectory(
pipConstructionHelper,
downloadData,
DirectoryArtifact.CreateWithZeroPartialSealId(downloadData.DownloadedFilePath.GetParent(m_context.PathTable)),
SortedReadOnlyArray <FileArtifact, OrdinalFileArtifactComparer> .FromSortedArrayUnsafe(
ReadOnlyArray <FileArtifact> .FromWithoutCopy(new[] { extractedFile }),
OrdinalFileArtifactComparer.Instance)));
}
Statistics.Extractions.Total.Increment();
using (Statistics.Extractions.UpToDateCheckDuration.Start(downloadData.Settings.Url))
{
var result = await CheckIfExtractIsNeededAsync(pipConstructionHelper, downloadData);
if (result.IsErrorValue)
{
Statistics.Extractions.Failures.Increment();
}
if (result != EvaluationResult.Continue)
{
Statistics.Extractions.SkippedDueToManifest.Increment();
return(result);
}
}
using (Statistics.Extractions.Duration.Start(downloadData.Settings.Url))
{
try
{
if (!await Task.Run(
() => TryExtractToDisk(downloadData),
m_context.CancellationToken))
{
Statistics.Extractions.Failures.Increment();
return(EvaluationResult.Error);
}
}
catch (TaskCanceledException)
{
return(EvaluationResult.Canceled);
}
}
using (Statistics.Extractions.UpToDateCheckDuration.Start(downloadData.Settings.Url))
{
var result = await ValidateAndStoreIncrementalExtractState(pipConstructionHelper, downloadData);
if (result.IsErrorValue)
{
Statistics.Extractions.Failures.Increment();
}
return(result);
}
}
/// <summary>
/// Creates a source file artifact.
/// </summary>
public FileArtifact CreateSourceFile(string relative) =>
FileArtifact.CreateSourceFile(m_sourceRoot.Combine(
Context.PathTable, RelativePath.Create(Context.StringTable, relative)));
public RelativePath GetCanonicalRelativePath(string rootPath, IEnumerable <string> relativePathParts)
{
return(RelativePath.CreateFromSegments(relativePathParts.ToArray()));
}
public void RootShouldReturnNull()
{
RelativePath parent = RelativePath.RootPath.Parent;
Assert.Null(parent);
}
public IList<RelativePath> GetDeletedEntries(RelativePath parentPath) {
return _deletedChildren.Value.GetValue(parentPath) ?? ArrayUtilities.EmptyList<RelativePath>.Instance;
}
/// <summary>
/// Converts <paramref name="value"/> to a <see cref="RelativePath"/>.
/// Returns false if the input is invalid.
/// </summary>
public static bool TryConvertRelativePath([NotNull] object value, [NotNull] StringTable stringTable, out RelativePath result)
{
if (value is RelativePath)
{
result = (RelativePath)value;
return(true);
}
var stringValue = value as string;
if (stringValue != null)
{
if (RelativePath.TryCreate(stringTable, stringValue, out result))
{
return(true);
}
}
result = RelativePath.Invalid;
return(false);
}
public bool IsChanged(RelativePath path) {
return GetPathChangeKind(path) == PathChangeKind.Changed;
}
private HashKey CreateAndStoreBlob(RelativePath filePath)
{
Blob blob = new Blob(filePath);
return(ObjectDatabase.Store(blob));
}
private string NormalizeHref(string href, RelativePath relativeToFile)
{
if (!Utility.IsSupportedRelativeHref(href))
{
return href;
}
return (relativeToFile + (RelativePath)href).GetPathFromWorkingFolder();
}
/// <summary>
/// Returns the node ids for a set of relative paths.
/// </summary>
/// <param name="session">An open session with the server to use.</param>
/// <param name="startNodeId">The starting node for the relative paths.</param>
/// <param name="namespacesUris">The namespace URIs referenced by the relative paths.</param>
/// <param name="relativePaths">The relative paths.</param>
/// <returns>A collection of local nodes.</returns>
public static List <NodeId> TranslateBrowsePaths(
Session session,
NodeId startNodeId,
NamespaceTable namespacesUris,
params string[] relativePaths)
{
// build the list of browse paths to follow by parsing the relative paths.
BrowsePathCollection browsePaths = new BrowsePathCollection();
if (relativePaths != null)
{
for (int ii = 0; ii < relativePaths.Length; ii++)
{
BrowsePath browsePath = new BrowsePath();
// The relative paths used indexes in the namespacesUris table. These must be
// converted to indexes used by the server. An error occurs if the relative path
// refers to a namespaceUri that the server does not recognize.
// The relative paths may refer to ReferenceType by their BrowseName. The TypeTree object
// allows the parser to look up the server's NodeId for the ReferenceType.
browsePath.RelativePath = RelativePath.Parse(
relativePaths[ii],
session.TypeTree,
namespacesUris,
session.NamespaceUris);
browsePath.StartingNode = startNodeId;
browsePaths.Add(browsePath);
}
}
// make the call to the server.
BrowsePathResultCollection results;
DiagnosticInfoCollection diagnosticInfos;
ResponseHeader responseHeader = session.TranslateBrowsePathsToNodeIds(
null,
browsePaths,
out results,
out diagnosticInfos);
// ensure that the server returned valid results.
Session.ValidateResponse(results, browsePaths);
Session.ValidateDiagnosticInfos(diagnosticInfos, browsePaths);
// collect the list of node ids found.
List <NodeId> nodes = new List <NodeId>();
for (int ii = 0; ii < results.Count; ii++)
{
// check if the start node actually exists.
if (StatusCode.IsBad(results[ii].StatusCode))
{
nodes.Add(null);
continue;
}
// an empty list is returned if no node was found.
if (results[ii].Targets.Count == 0)
{
nodes.Add(null);
continue;
}
// Multiple matches are possible, however, the node that matches the type model is the
// one we are interested in here. The rest can be ignored.
BrowsePathTarget target = results[ii].Targets[0];
if (target.RemainingPathIndex != UInt32.MaxValue)
{
nodes.Add(null);
continue;
}
// The targetId is an ExpandedNodeId because it could be node in another server.
// The ToNodeId function is used to convert a local NodeId stored in a ExpandedNodeId to a NodeId.
nodes.Add(ExpandedNodeId.ToNodeId(target.TargetId, session.NamespaceUris));
}
// return whatever was found.
return(nodes);
}
public static ConstructedCommand <TSpec> Create <TSpec>(TSpec command, RelativePath OutputLocation = null)
where TSpec : CommandSpec <TSpec>, new()
=> new ConstructedCommand <TSpec>(command, OutputLocation);
public ConstructedCommand(ICommandSpec Command, RelativePath OutputLocation = null)
{
this.Command = Command;
this.OutputLocation = OutputLocation ?? string.Empty;
}
public void Release(bool unloadAllLoadedObjects) //, bool forceUnloadAsset = false
{
if (unloadAllLoadedObjects && m_object) //&& (forceUnloadAsset || (!m_neverUnloadAsset && !forceUnloadAsset))
{
//UnloadAsset();
if (!RelativePath.EndsWith(".prefab", StringComparison.OrdinalIgnoreCase) && !RelativePath.EndsWith(".fbx", StringComparison.OrdinalIgnoreCase))
{
Resources.UnloadAsset(m_object);
}
m_object = null;
Debug.LogWarning("set m_object null: " + RelativePath);//Debug.LogWarning
}
//if (SystemSwitch.UseFileSystem)
//{
// if (this.createRequest != null)
// {
// if (this.createRequest.assetBundle)
// this.createRequest.assetBundle.Unload(unloadAllLoadedObjects);
// this.createRequest = null;
// }
//}
//else
//{
if (this.www != null)
{
if (this.www.assetBundle)
{
this.www.assetBundle.Unload(unloadAllLoadedObjects);
}
this.www = null;
}
//}
}
public static async Task Initialize(TestContext _)
{
string sourceDocumentPath = RelativePath.GetFilePath($"{nameof(ArrayParameterTestCases)}.cs");
(project, sourceDocumentId) = await Workspace.GetCurrentProjectWithDocumentAsync(sourceDocumentPath);
}
/// <summary>
/// Initializes a new instance of the class.
/// </summary>
public RemoteLogBox(ListBox listBox, bool colorize = false)
: base(listBox, colorize)
{
agentClient = null;
logPath = new RelativePath();
}
private ParsedPath ParseAbsoluteOrRelativePath(StringSegment fragment, bool isPathFragment)
{
if (fragment.Length == 0)
{
return(ParsedPath.Invalid());
}
// If we're running on Unix skip all of this, because absolute paths are always prefixed with '/'
if (fragment[0] == '/' && !OperatingSystemHelper.IsUnixOS)
{
if (isPathFragment)
{
// Relative paths can't start with '/'
return(ParsedPath.Invalid());
}
if (fragment.Length < 2)
{
// Path '/' is invalid.
return(ParsedPath.Invalid());
}
if (fragment[1] != '/' && fragment[1] != '?')
{
// Case: '/f/g/h'.
var relativePath = fragment.Subsegment(1, fragment.Length - 1);
if (!RelativePath.TryCreate(m_pathTable.StringTable, relativePath, out RelativePath part))
{
return(ParsedPath.Invalid());
}
// Contract.Assert(m_rootPath.IsValid, "To get absolute path root path should be valid");
return(ParsedPath.PackageRelativePath(part, m_pathTable));
}
}
try
{
if (IsPathRooted(fragment))
{
if (isPathFragment)
{
// Relative paths can't be rooted.
return(ParsedPath.Invalid());
}
// Case: 'c:/f/g/h'.
if (!AbsolutePath.TryCreate(m_pathTable, fragment, out AbsolutePath absolutePath))
{
return(ParsedPath.Invalid());
}
return(ParsedPath.AbsolutePath(absolutePath, m_pathTable));
}
}
catch (ArgumentException)
{
return(ParsedPath.Invalid());
}
// Case: 'f/g/h', or '../f/g/h', or './f/g/h'.
if (isPathFragment)
{
// .. is invalid in path fragments
if (!RelativePath.TryCreate(m_pathTable.StringTable, fragment, out RelativePath part))
{
return(ParsedPath.Invalid());
}
return(ParsedPath.FileRelativePath(part, 0, m_pathTable));
}
int index = 0;
int parentCount = 0;
while (CheckAndUpdatePathFragment(fragment, ref index, out bool getParent))
{
if (getParent)
{
parentCount++;
}
}
var relativePathFragment = fragment;
if (index == 0)
{
// Do nothing.
}
else if (index >= fragment.Length)
{
relativePathFragment = ".";
}
else
{
relativePathFragment = fragment.Subsegment(index, fragment.Length - index);
}
if (!RelativePath.TryCreate(m_pathTable.StringTable, relativePathFragment, out RelativePath relativePart))
{
return(ParsedPath.Invalid());
}
return(ParsedPath.FileRelativePath(relativePart, parentCount, m_pathTable));
}
public override int GetHashCode()
{
return(RelativePath.GetHashCode());
}
public void PathUsesNormalSeparator()
{
var path = RelativePath.CreateFromSegments("a", "b", "c");
Assert.That(path.ToString(), Is.EqualTo("a/b/c"));
}
