public static int Main(string[] args)
{
var fileSystem = new FileSystem();
string srcDirectory = AppDomain.CurrentDomain.BaseDirectory.ParentDirectory().ParentDirectory().ParentDirectory().ParentDirectory();
string buildDirectory = args.Any()
? args.Single()
: srcDirectory.ParentDirectory().ParentDirectory().AppendPath("buildsupport");
Console.WriteLine("Trying to copy to " + buildDirectory);
if (!fileSystem.DirectoryExists(buildDirectory))
{
throw new ApplicationException("Could not find the buildsupport directory");
}
var targetDirectory = buildDirectory.AppendPath("FubuDocs");
fileSystem.CreateDirectory(targetDirectory);
fileSystem.CleanDirectory(targetDirectory);
var fromDirectory = srcDirectory.AppendPath("FubuDocsRunner").AppendPath("bin").AppendPath("debug");
var files = FileSet.Deep("*.dll;*.pdb;*.exe");
fileSystem.FindFiles(fromDirectory, files).Each(file => {
Console.WriteLine("Copying {0} to {1}", file, targetDirectory);
fileSystem.Copy(file, targetDirectory, CopyBehavior.overwrite);
});
return 0;
}
public override void LaunchEditor(FileSystem fs, File file)
{
var data = file.GetData();
var ms = new MemoryStream(data);
var modelFile = new ModelFile();
try
{
modelFile.Open(ms);
}
finally
{
ms.Close();
}
if (modelFile.EmbeddedTextureFile != null)
{
ShowForm(file, modelFile.EmbeddedTextureFile);
}
else
{
MessageBox.Show("There are no embedded textures in the selected model file to edit.", "Edit",
MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
}
}
public void DeployHelloWorld()
{
IFileSystem fileSystem = new FileSystem();
var settings = new DeploymentSettings(@"C:\dev\test-profile\");
IBottleRepository bottles = new BottleRepository(fileSystem, new PackageExploder(new ZipFileService(fileSystem), new PackageExploderLogger(s=>{ }), fileSystem ), settings);
var initializer = new IisFubuInitializer(fileSystem, new DeploymentSettings());
var deployer = new IisFubuDeployer(fileSystem, bottles);
var directive = new FubuWebsite();
directive.HostBottle = "test";
directive.WebsiteName = "fubu";
directive.WebsitePhysicalPath = @"C:\dev\test-web";
directive.VDir = "bob";
directive.VDirPhysicalPath = @"C:\dev\test-app";
directive.AppPool = "fubizzle";
directive.DirectoryBrowsing = Activation.Enable;
initializer.Execute(directive, new HostManifest("something"), new PackageLog());
var host = new HostManifest("a");
deployer.Execute(directive, host, new PackageLog());
}
public void SetUp()
{
fileSystem = Substitute.For<FileSystem>((Config)null);
fileSystem.ReadXmlFile().Returns(Resource.SimpleSource);
timeContext = Substitute.For<TimeContext>();
sut = new XmlDataModificator(timeContext, fileSystem);
}
public ObjectCreator(Manifest manifest, FileSystem.FileSystem modFiles)
{
typeCache = new Cache<string, Type>(FindType);
ctorCache = new Cache<Type, ConstructorInfo>(GetCtor);
// Allow mods to load types from the core Game assembly, and any additional assemblies they specify.
var assemblyList = new List<Assembly>() { typeof(Game).Assembly };
foreach (var path in manifest.Assemblies)
{
var data = modFiles.Open(path).ReadAllBytes();
// .NET doesn't provide any way of querying the metadata of an assembly without either:
// (a) loading duplicate data into the application domain, breaking the world.
// (b) crashing if the assembly has already been loaded.
// We can't check the internal name of the assembly, so we'll work off the data instead
var hash = CryptoUtil.SHA1Hash(data);
Assembly assembly;
if (!ResolvedAssemblies.TryGetValue(hash, out assembly))
{
assembly = Assembly.Load(data);
ResolvedAssemblies.Add(hash, assembly);
}
assemblyList.Add(assembly);
}
AppDomain.CurrentDomain.AssemblyResolve += ResolveAssembly;
assemblies = assemblyList.SelectMany(asm => asm.GetNamespaces().Select(ns => Pair.New(asm, ns))).ToArray();
AppDomain.CurrentDomain.AssemblyResolve -= ResolveAssembly;
}
private static void RefreshAssemblies(ILog log)
{
var fileSystem = new FileSystem();
var packagesFolder = Path.Combine(fileSystem.CurrentDirectory, "packages");
if(fileSystem.DirectoryExists(packagesFolder))
{
// Delete any blacklisted packages to avoid various issues with PackageAssemblyResolver
// https://github.com/scriptcs/scriptcs/issues/511
foreach (var packagePath in
_blacklistedPackages.SelectMany(packageName => Directory.GetDirectories(packagesFolder)
.Where(d => new DirectoryInfo(d).Name.StartsWith(packageName, StringComparison.InvariantCultureIgnoreCase)),
(packageName, packagePath) => new {packageName, packagePath})
.Where(t => fileSystem.DirectoryExists(t.packagePath))
.Select(t => @t.packagePath))
{
fileSystem.DeleteDirectory(packagePath);
}
}
var par = new PackageAssemblyResolver(fileSystem, new PackageContainer(fileSystem, log), log);
_assemblies = par.GetAssemblyNames(fileSystem.CurrentDirectory).ToList();
// Add the assemblies in the current directory
_assemblies.AddRange(Directory.GetFiles(fileSystem.CurrentDirectory, "*.dll")
.Where(a => new AssemblyUtility().IsManagedAssembly(a)));
}
public static void Extract(this ZipArchive archive, FileSystem fileSystem, string directoryName)
{
foreach (ZipArchiveEntry entry in archive.Entries)
{
string path = Path.Combine(directoryName, entry.FullName);
if (entry.Length == 0 && (path.EndsWith("/", StringComparison.Ordinal) || path.EndsWith("\\", StringComparison.Ordinal)))
{
// Extract directory
fileSystem.Directory.CreateDirectory(path);
}
else
{
FileInfoBase fileInfo = fileSystem.FileInfo.FromFileName(path);
if (!fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
using (Stream zipStream = entry.Open(),
fileStream = fileInfo.OpenWrite())
{
zipStream.CopyTo(fileStream);
}
}
}
}
private static JasperServiceConfiguration serviceConfiguration()
{
var directory = FubuRuntime.DefaultApplicationPath();
var fileSystem = new FileSystem();
return fileSystem.LoadFromFile<JasperServiceConfiguration>(directory, JasperServiceConfiguration.FILE);
}
public static PersistenceExpression<Solution> ForSolution(Solution target)
{
var file = "{0}-{1}.config".ToFormat(typeof(Solution).Name, Guid.NewGuid());
var fileSystem = new FileSystem();
if (fileSystem.FileExists(file))
{
fileSystem.DeleteFile(file);
}
var writer = ObjectBlockWriter.Basic(new RippleBlockRegistry());
var contents = writer.Write(target);
Debug.WriteLine(contents);
fileSystem.WriteStringToFile(file, contents);
var reader = SolutionLoader.Reader();
var specification = new PersistenceSpecification<Solution>(x =>
{
var fileContents = fileSystem.ReadStringFromFile(file);
var readValue = Solution.Empty();
reader.Read(readValue, fileContents);
fileSystem.DeleteFile(file);
return readValue;
});
specification.Original = target;
return new PersistenceExpression<Solution>(specification);
}
public MixFile(FileSystem context, string filename)
{
Name = filename;
this.context = context;
s = context.Open(filename);
try
{
// Detect format type
var isCncMix = s.ReadUInt16() != 0;
// The C&C mix format doesn't contain any flags or encryption
var isEncrypted = false;
if (!isCncMix)
isEncrypted = (s.ReadUInt16() & 0x2) != 0;
List<PackageEntry> entries;
if (isEncrypted)
{
long unused;
entries = ParseHeader(DecryptHeader(s, 4, out dataStart), 0, out unused);
}
else
entries = ParseHeader(s, isCncMix ? 0 : 4, out dataStart);
index = ParseIndex(entries.ToDictionaryWithConflictLog(x => x.Hash,
"{0} ({1} format, Encrypted: {2}, DataStart: {3})".F(filename, isCncMix ? "C&C" : "RA/TS/RA2", isEncrypted, dataStart),
null, x => "(offs={0}, len={1})".F(x.Offset, x.Length)));
}
catch (Exception)
{
Dispose();
throw;
}
}
public void DoNotCompressAlreadyCompressedFiles()
{
// Arrange
//FileHandler fileHandler = new JpgFileHandler();
var fileSystem = new FileSystem();
fileSystem.DeleteDirectoryAndAllFiles(TestConstants.NewDirectory);
fileSystem.DeleteDirectoryAndAllFiles(TestConstants.TempDirectory);
IFileHandler fileHandler = _fileHandlerFactory.GetFileHandler(TestConstants.ExistingJpgFullFileName, fileSystem);
IPictureDirectory tempDir = _directoryFactory.GetOrCreateDirectory(TestConstants.TempDirectory);
IPictureDirectory newDir = _directoryFactory.GetOrCreateDirectory(TestConstants.NewDirectory);
string originalFileName = fileHandler.FileName;
string tempFileName = fileHandler.PerformRenameAndMove(tempDir);
string compressedFileName = fileHandler.PerformCompressAndMove(newDir);
bool caughtExpectedException = false;
// Act
try {
string compressedFileName2 = fileHandler.PerformCompressAndMove(tempDir);
} catch (IOException) {
caughtExpectedException = true;
}
// Assert
Assert.IsTrue(caughtExpectedException);
}
public UncDirectoryProvider([NotNull] FileSystem fileSystem)
: base(fileSystem)
{
Assert.ArgumentNotNull(fileSystem, "fileSystem");
this.fileSystem = fileSystem;
}
public void can_read_and_write_the_packages_config()
{
var theFileSystem = new FileSystem();
theFileSystem.WriteStringToFile(NuGetDependencyStrategy.PackagesConfig, "<?xml version=\"1.0\" encoding=\"utf-8\"?><packages></packages>");
var theSolution = new Solution();
theSolution.AddDependency(new Dependency("Bottles", "1.0.1.1"));
theSolution.AddDependency(new Dependency("FubuCore", "1.2.0.1"));
var theProject = new Project("Test.csproj");
theProject.AddDependency("Bottles");
theProject.AddDependency("FubuCore");
theSolution.AddProject(theProject);
var theStrategy = new NuGetDependencyStrategy();
theStrategy.Write(theProject);
theStrategy
.Read(theProject)
.ShouldHaveTheSameElementsAs(
new Dependency("Bottles", "1.0.1.1"),
new Dependency("FubuCore", "1.2.0.1")
);
theFileSystem.DeleteFile(NuGetDependencyStrategy.PackagesConfig);
}
public void GenerateDeployScript_should_generate_correct_script_on_disk()
{
var generator = new SqlScriptGenerator();
SqlScriptingInput input = generator.GetSqlScriptingInput(@"dsql02.ada-dev.nl\v2012", @"SMS");
input.RootFolder = @"C:\Temp\Generated";
SqlDeployScriptResult result = generator.GenerateDeployScript(input);
string template = Assembly.GetExecutingAssembly().GetFileAsString("Research.EndToEndTests.CodeGeneration.SqlScriptGeneratorTester_GenerateDatabaseDeployScript_result.cmd");
string content = string.Format(template, result.Schemas, result.Tables, result.Functions, result.StoredProcedures);
var file = new FileInfoDto
{
Path = Path.Combine(input.RootFolder, "Deploy.cmd")
};
file.Content = content;
var fs = new FileSystem();
Task deleteDirectoryTask = fs.DeleteDirectoryAsync(input.RootFolder);
deleteDirectoryTask.Wait();
Task generateFilesTask = fs.GenerateFileAsync(file);
generateFilesTask.Wait();
Assert.AreEqual(true, true);
}
public LuxConfigHost(IInternalConfigHost internalConfigHost)
{
FileSystem = new FileSystem();
if (ConfigSystemProxy.FileSystem != null)
FileSystem = ConfigSystemProxy.FileSystem;
Host = internalConfigHost;
}
public static void CreateFileFromResourceName(string resourceName, string path)
{
var fileSystem = new FileSystem();
using (var resourceStream = GetResourceStreamFromResourceName(resourceName)) {
fileSystem.WriteStreamToFile(path, resourceStream);
}
}
public void LoadAllProjects()
{
var fileSystem = new FileSystem();
var folder = AppContext.BaseDirectory;
while (!folder.EndsWith("Marten.Testing"))
{
folder = folder.ParentDirectory();
}
folder = folder.AppendPath("CodeTracker");
var files = fileSystem.FindFiles(folder, FileSet.Shallow("*.json"));
AllProjects = new Dictionary<Guid, GithubProject>();
foreach (var file in files)
{
var project = GithubProject.LoadFrom(file);
AllProjects.Add(project.Id, project);
Console.WriteLine($"Loaded {project.OrganizationName}{project.ProjectName} from JSON");
}
_store.Advanced.Clean.DeleteAllDocuments();
PublishAllProjectEvents(_store);
}
public void should_write_out_the_second_sample()
{
var sample2 = new FileSystem().ReadStringFromFile("samples", "Sample2.txt");
sample2.ShouldContain("private string otherName;");
sample2.ShouldContain("private string moreVariables;");
sample2.ShouldContain("private int order;");
}
public void generate_visualizers()
{
var path = @"C:\code\diagnostics\src\FubuMVC.Diagnostics\Visualization\Visualizers\ConfigurationActions";
var types = typeof (FubuRequest).Assembly.GetExportedTypes()
.Where(x => x.IsConcreteTypeOf<IConfigurationAction>());
var fileSystem = new FileSystem();
types.Each(x =>
{
var writer = new StringWriter();
/*
<use master="" />
<viewdata model="FubuMVC.Diagnostics.Visualization.BehaviorNodeViewModel" />
*/
writer.WriteLine("<use master=\"\" />");
writer.WriteLine("<viewdata model=\"{0}\" />", x.FullName);
var file = path.AppendPath(x.Name + ".spark");
fileSystem.WriteStringToFile(file, writer.ToString());
});
}
public void Setup()
{
_testSubject = MakeTestSubject();
_testSubject.EnableRevertToHere();
_runRootFolder = _testSubject.TempDirectory.Result.Dir("CreatedByTestRun-" + Guid.NewGuid());
FinishSetup();
}
public LocalScriptStore(LoggerConfigurator logConfig, FileSystem fileSystem, IRobotPluginLocator pluginLocator)
{
_fileSystem = fileSystem;
_pluginLocator = pluginLocator;
_scriptUpdated = new Subject<IScript>();
_log = logConfig.GetLogger();
}
private void LoadSampleCommands()
{
var fileSystem = new FileSystem();
var commandSymbols = new List<string> { ":", "#" };
var fileContents = new StringBuilder();
var path = fileSystem.GetWorkingDirectory(fileSystem.CurrentDirectory);
var files = fileSystem.EnumerateFiles(path, "*.scriptcommands", SearchOption.TopDirectoryOnly);
foreach (var fileName in files.Select(file => Path.Combine(fileSystem.CurrentDirectory, file)))
{
using (TextReader textReader = new StreamReader(fileName))
{
fileContents.AppendFormat("{0}", textReader.ReadToEnd());
}
}
var commandstoParse = fileContents.ToString();
if (string.IsNullOrWhiteSpace(commandstoParse)) return;
var commandScripts = SplitLines(commandstoParse);
foreach (var command in commandScripts.Where(command => !string.IsNullOrWhiteSpace(command)))
{
var commandNameLength = command.IndexOf("]", StringComparison.Ordinal);
var commandName = command.Substring(0, commandNameLength);
var commandScript = command.Substring(commandNameLength + 1);
ReplCommandPackUtils.MapCommand(SampleCommands, commandName, commandScript, commandSymbols);
}
}
public BigFile(FileSystem context, string filename)
{
Name = filename;
s = context.Open(filename);
try
{
if (s.ReadASCII(4) != "BIGF")
throw new InvalidDataException("Header is not BIGF");
// Total archive size.
s.ReadUInt32();
var entryCount = s.ReadUInt32();
if (BitConverter.IsLittleEndian)
entryCount = int2.Swap(entryCount);
// First entry offset? This is apparently bogus for EA's .big files
// and we don't have to try seeking there since the entries typically start next in EA's .big files.
s.ReadUInt32();
for (var i = 0; i < entryCount; i++)
{
var entry = new Entry(s);
index.Add(entry.Path, entry);
}
}
catch
{
Dispose();
throw;
}
}
virtual protected void Access(String Path, out FileSystem NewFileSystem, out String NewPath)
{
// Normalize Components
Path = AbsoluteNormalizePath(Path, CurrentWorkingPath);
var ComparePath = ComparablePath(Path);
// Check MountedFileSystems.
foreach (var Item in MountedFileSystems)
{
var CheckMountedPath = ComparablePath(Item.Key);
var MountInfo = Item.Value;
if (
ComparePath.StartsWith(CheckMountedPath) &&
(
(CheckMountedPath.Length == ComparePath.Length) ||
(ComparePath.Substring(CheckMountedPath.Length, 1) == "/")
)
) {
var NewAccessPath = ComparePath.Substring(CheckMountedPath.Length);
if (MountInfo.Path != "/")
{
NewAccessPath = MountInfo.Path + "/" + NewAccessPath;
}
// Use Mounted File System.
MountInfo.FileSystem.Access(NewAccessPath, out NewFileSystem, out NewPath);
return;
}
}
NewFileSystem = this;
NewPath = Path;
}
public IDirectory CreateChildDirectory(string name)
{
var newPath = System.IO.Path.Combine(Path, name);
var fileSystem = new FileSystem();
fileSystem.CreateDirectory(newPath);
return fileSystem.GetDirectory(newPath);
}
public void Execute(string[] arguments)
{
Settings.Parse(arguments);
var fileSystem = new FileSystem();
if (string.IsNullOrWhiteSpace(Path))
{
Path = Directory.GetCurrentDirectory();
}
if (string.IsNullOrWhiteSpace(Engine))
{
Engine = InferEngineFromDirectory(Path, fileSystem);
}
ISiteEngine engine;
if (engineMap.TryGetValue(Engine, out engine))
{
var context = new SiteContext { Folder = Path };
engine.Initialize(fileSystem, context);
engine.Process();
}
else
{
System.Diagnostics.Debug.WriteLine("Cannot find engine for input: '{0}'", Engine);
}
}
public AsyncDaemonFixture()
{
_store = TestingDocumentStore.For(_ =>
{
_.DatabaseSchemaName = "expected";
_.Events.DatabaseSchemaName = "expected";
_.Events.InlineProjections.AggregateStreamsWith<ActiveProject>();
_.Events.InlineProjections.TransformEvents(new CommitViewTransform());
});
var folder = ".".ToFullPath().ParentDirectory().ParentDirectory()
.AppendPath("CodeTracker");
var files = new FileSystem().FindFiles(folder, FileSet.Shallow("*.json"));
AllProjects = new Dictionary<Guid, GithubProject>();
foreach (var file in files)
{
var project = GithubProject.LoadFrom(file);
AllProjects.Add(project.Id, project);
Console.WriteLine($"Loaded {project.OrganizationName}{project.ProjectName} from JSON");
}
PublishAllProjectEvents(_store);
}
private static BottleServiceConfiguration serviceConfiguration()
{
var directory = BottlesServicePackageFacility.GetApplicationDirectory();
var fileSystem = new FileSystem();
return fileSystem.LoadFromFile<BottleServiceConfiguration>(directory, BottleServiceConfiguration.FILE);
}
public void SetUp()
{
var fileSystem = new FileSystem();
fileSystem.Copy("FubuMVC.SlickGrid.Docs.csproj.fake", "FubuMVC.SlickGrid.Docs.csproj");
fileSystem.Copy("SlickGridHarness.csproj.fake", "SlickGridHarness.csproj");
}
public void RenameClasses()
{
var folder = ".".ToFullPath().ParentDirectory().ParentDirectory()
.AppendPath("CodeTracker");
var fileSystem = new FileSystem();
var files = fileSystem.FindFiles(folder, FileSet.Shallow("*.json"));
foreach (var file in files)
{
var json = fileSystem.ReadStringFromFile(file);
json = replace(json, "GithubProject");
json = replace(json, "Timestamped[]");
json = replace(json, "ProjectStarted");
json = replace(json, "IssueCreated");
json = replace(json, "IssueClosed");
json = replace(json, "IssueReopened");
json = replace(json, "Commit");
fileSystem.WriteStringToFile(file, json);
}
}
public void TearDown()
{
new FileSystem().DeleteFile(FileSystem.Combine(theApplicationDirectory, PackageManifest.FILE));
}
internal void LoadIniFile(string filePath)
{
LoadIniFile(FileSystem.GetFile(filePath));
}
public async Task PutFileRequiringNewReplicaCloseToHardLimitDoesNotHang()
{
var context = new Context(Logger);
using (DisposableDirectory testDirectory = new DisposableDirectory(FileSystem))
{
#pragma warning disable AsyncFixer04 // A disposable object used in a fire & forget async call
Task testTask = TestStore(context, Clock, testDirectory, async store =>
#pragma warning restore AsyncFixer04 // A disposable object used in a fire & forget async call
{
// Make a file which will overflow the cache size with just 2 copies.
PutResult putResult = await store.PutRandomAsync(context, ContentSizeToStartHardPurging(2));
ResultTestExtensions.ShouldBeSuccess((BoolResult)putResult);
ContentHash hash = putResult.ContentHash;
AbsolutePath primaryPath = testDirectory.Path / "hardlinkPrimary.txt";
// Hardlink the file out once.
PlaceFileResult placeResult = await store.PlaceFileAsync(
context,
hash,
primaryPath,
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.HardLink,
null);
Assert.True(placeResult.Succeeded);
// Hardlink the file out 1023 more times. Since the limit is 1024 total, and we already had 2,
// this will overflow the links and cause the CAS to create a new replica for it. This will cause
// the purger to consider that hash for eviction *while making room for that hash*, which is the
// trigger for the previous deadlock that this test will now guard against.
for (int i = 0; i < 1023; i++)
{
AbsolutePath newPath = testDirectory.Path / $"hardlink{i}.txt";
await FileSystem.CopyFileAsync(primaryPath, newPath, false);
putResult = await store.PutFileAsync(
context,
newPath,
FileRealizationMode.HardLink,
hash,
null);
// The checks below are just to make sure that the calls completed as expected.
// The most important part is that they complete *at all*, which is enforced by
// racing against the Task.Delay in the outer scope.
if (i < 1022 || SucceedsEvenIfFull)
{
// The first 1022 links should succeed (bringing it up to the limit of 1024)
// And *all* of the calls should succeed if the cache takes new content even when overflowed.
Assert.True(putResult.Succeeded);
}
else
{
// If the implementation rejects overflowing content, then the last call should fail.
Assert.False(putResult.Succeeded);
Assert.Contains("Failed to reserve space", putResult.ErrorMessage);
}
}
});
// Race between the test and a 1-minute timer. This can be increased if the test ends up flaky.
Task firstCompletedTask = await Task.WhenAny(testTask, Task.Delay(TimeSpan.FromMinutes(1)));
// The test should finish first, long before a minute passes, but it won't if it deadlocks.
Assert.True(firstCompletedTask == testTask);
await firstCompletedTask;
}
}
public void Delete(bool recursive) => FileSystem.RemoveDirectory(FullPath, recursive);
/// <summary>
/// Starts the FFMPEG.
/// </summary>
/// <param name="state">The state.</param>
/// <param name="outputPath">The output path.</param>
/// <param name="cancellationTokenSource">The cancellation token source.</param>
/// <param name="workingDirectory">The working directory.</param>
/// <returns>Task.</returns>
protected async Task <TranscodingJob> StartFfMpeg(
StreamState state,
string outputPath,
CancellationTokenSource cancellationTokenSource,
string workingDirectory = null)
{
Directory.CreateDirectory(Path.GetDirectoryName(outputPath));
await AcquireResources(state, cancellationTokenSource).ConfigureAwait(false);
if (state.VideoRequest != null && !string.Equals(state.OutputVideoCodec, "copy", StringComparison.OrdinalIgnoreCase))
{
var auth = AuthorizationContext.GetAuthorizationInfo(Request);
if (auth.User != null)
{
if (!auth.User.Policy.EnableVideoPlaybackTranscoding)
{
ApiEntryPoint.Instance.OnTranscodeFailedToStart(outputPath, TranscodingJobType, state);
throw new ArgumentException("User does not have access to video transcoding");
}
}
}
var encodingOptions = ApiEntryPoint.Instance.GetEncodingOptions();
var transcodingId = Guid.NewGuid().ToString("N");
var commandLineArgs = GetCommandLineArguments(outputPath, encodingOptions, state, true);
var process = new Process()
{
StartInfo = new ProcessStartInfo()
{
WindowStyle = ProcessWindowStyle.Hidden,
CreateNoWindow = true,
UseShellExecute = false,
// Must consume both stdout and stderr or deadlocks may occur
//RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true,
FileName = MediaEncoder.EncoderPath,
Arguments = commandLineArgs,
WorkingDirectory = string.IsNullOrWhiteSpace(workingDirectory) ? null : workingDirectory,
ErrorDialog = false
},
EnableRaisingEvents = true
};
var transcodingJob = ApiEntryPoint.Instance.OnTranscodeBeginning(outputPath,
state.Request.PlaySessionId,
state.MediaSource.LiveStreamId,
transcodingId,
TranscodingJobType,
process,
state.Request.DeviceId,
state,
cancellationTokenSource);
var commandLineLogMessage = process.StartInfo.FileName + " " + process.StartInfo.Arguments;
Logger.LogInformation(commandLineLogMessage);
var logFilePrefix = "ffmpeg-transcode";
if (state.VideoRequest != null)
{
if (string.Equals(state.OutputVideoCodec, "copy", StringComparison.OrdinalIgnoreCase) &&
string.Equals(state.OutputAudioCodec, "copy", StringComparison.OrdinalIgnoreCase))
{
logFilePrefix = "ffmpeg-directstream";
}
else if (string.Equals(state.OutputVideoCodec, "copy", StringComparison.OrdinalIgnoreCase))
{
logFilePrefix = "ffmpeg-remux";
}
}
var logFilePath = Path.Combine(ServerConfigurationManager.ApplicationPaths.LogDirectoryPath, logFilePrefix + "-" + Guid.NewGuid() + ".txt");
Directory.CreateDirectory(Path.GetDirectoryName(logFilePath));
// FFMpeg writes debug/error info to stderr. This is useful when debugging so let's put it in the log directory.
state.LogFileStream = FileSystem.GetFileStream(logFilePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, true);
var commandLineLogMessageBytes = Encoding.UTF8.GetBytes(Request.AbsoluteUri + Environment.NewLine + Environment.NewLine + JsonSerializer.SerializeToString(state.MediaSource) + Environment.NewLine + Environment.NewLine + commandLineLogMessage + Environment.NewLine + Environment.NewLine);
await state.LogFileStream.WriteAsync(commandLineLogMessageBytes, 0, commandLineLogMessageBytes.Length, cancellationTokenSource.Token).ConfigureAwait(false);
process.Exited += (sender, args) => OnFfMpegProcessExited(process, transcodingJob, state);
try
{
process.Start();
}
catch (Exception ex)
{
Logger.LogError(ex, "Error starting ffmpeg");
ApiEntryPoint.Instance.OnTranscodeFailedToStart(outputPath, TranscodingJobType, state);
throw;
}
// MUST read both stdout and stderr asynchronously or a deadlock may occurr
//process.BeginOutputReadLine();
state.TranscodingJob = transcodingJob;
// Important - don't await the log task or we won't be able to kill ffmpeg when the user stops playback
new JobLogger(Logger).StartStreamingLog(state, process.StandardError.BaseStream, state.LogFileStream);
// Wait for the file to exist before proceeeding
while (!File.Exists(state.WaitForPath ?? outputPath) && !transcodingJob.HasExited)
{
await Task.Delay(100, cancellationTokenSource.Token).ConfigureAwait(false);
}
if (state.IsInputVideo && transcodingJob.Type == TranscodingJobType.Progressive && !transcodingJob.HasExited)
{
await Task.Delay(1000, cancellationTokenSource.Token).ConfigureAwait(false);
if (state.ReadInputAtNativeFramerate && !transcodingJob.HasExited)
{
await Task.Delay(1500, cancellationTokenSource.Token).ConfigureAwait(false);
}
}
if (!transcodingJob.HasExited)
{
StartThrottler(state, transcodingJob);
}
return(transcodingJob);
}
static void DoIndexing()
{
int count_dirs = 0;
int count_files = 0;
Indexable indexable;
pending_request = new IndexerRequest();
Queue modified_directories = new Queue();
while (pending_directories.Count > 0)
{
DirectoryInfo dir = (DirectoryInfo)pending_directories.Dequeue();
AddToRequest(DirectoryToIndexable(dir, modified_directories));
try {
if (arg_recursive)
{
foreach (DirectoryInfo subdir in DirectoryWalker.GetDirectoryInfos(dir))
{
if (!Ignore(subdir) &&
!FileSystem.IsSpecialFile(subdir.FullName))
{
pending_directories.Enqueue(subdir);
}
}
}
foreach (FileInfo file in DirectoryWalker.GetFileInfos(dir))
{
if (!Ignore(file) &&
!FileSystem.IsSpecialFile(file.FullName))
{
AddToRequest(FileToIndexable(file));
count_files++;
}
}
} catch (DirectoryNotFoundException) {}
if (Shutdown.ShutdownRequested)
{
break;
}
count_dirs++;
}
Logger.Log.Debug("Scanned {0} files and directories in {1} directories", count_dirs + count_files, count_dirs);
if (Shutdown.ShutdownRequested)
{
backing_fa_store.Flush();
return;
}
// Time to remove deleted directories from the index and attributes store
while (modified_directories.Count > 0)
{
DirectoryInfo subdir = (DirectoryInfo)modified_directories.Dequeue();
Logger.Log.Debug("Checking {0} for deleted files and directories", subdir.FullName);
// Get a list of all documents from lucene index with ParentDirUriPropKey set as that of subdir
ICollection all_dirent = GetAllItemsInDirectory(subdir);
foreach (Dirent info in all_dirent)
{
// check if the item exists
if ((!info.IsDirectory && File.Exists(info.FullName)) ||
(info.IsDirectory && Directory.Exists(info.FullName)))
{
continue;
}
if (info.IsDirectory)
{
// Recursively remove deleted subdirectories
modified_directories.Enqueue(new DirectoryInfo(info.FullName));
}
// remove
Uri uri = PathToUri(info.FullName);
indexable = new Indexable(IndexableType.Remove, uri);
AddToRequest(indexable);
}
}
bool reschedule = false;
// Call Flush until our request is empty. We have to do this in a loop
// because Flush happens in a batch size and some indexables might generate more indexables
while (reschedule || pending_request.Count > 0)
{
if (Shutdown.ShutdownRequested)
{
break;
}
reschedule = FlushIndexer(driver);
}
backing_fa_store.Flush();
if (Shutdown.ShutdownRequested)
{
return;
}
Logger.Log.Debug("Optimizing index");
driver.OptimizeNow();
}
internal VersionControlSystem(IVersionControlSystemOptions options, ITerminal terminal)
{
Options = options;
Terminal = terminal;
_repoRoot = new Lazy <string>(() => !Options.RepoRoot.RemoveAllWhiteSpace().Equals(string.Empty) ? FileSystem.NormalizedPath(Options.RepoRoot) : FileSystem.NormalizedPath("."));
_sourceCode = new Lazy <IEnumerable <string> >(LoadSourceCode);
}
/// <summary>
/// Gets the file system root folder from the underlying
/// file system.
/// </summary>
public OperationResult <VirtualFolderInfo> Get()
{
return(SecureFunc(() => FileSystem.GetFileSystemRoot()));
}
private static IFileSystem CreateFileSystem(ScriptApplicationHostOptions hostOptions, string hostJsonContent = null)
{
string rootPath = hostOptions.ScriptPath;
string testDataPath = hostOptions.TestDataPath;
var fullFileSystem = new FileSystem();
var fileSystem = new Mock <IFileSystem>();
var fileBase = new Mock <FileBase>();
var dirBase = new Mock <DirectoryBase>();
fileSystem.SetupGet(f => f.Path).Returns(fullFileSystem.Path);
fileSystem.SetupGet(f => f.File).Returns(fileBase.Object);
fileBase.Setup(f => f.Exists(Path.Combine(rootPath, "host.json"))).Returns(true);
hostJsonContent = hostJsonContent ?? @"{ ""durableTask"": { ""HubName"": ""TestHubValue"", ""azureStorageConnectionStringName"": ""DurableStorage"" }}";
var testHostJsonStream = new MemoryStream(Encoding.UTF8.GetBytes(hostJsonContent));
testHostJsonStream.Position = 0;
fileBase.Setup(f => f.Open(Path.Combine(rootPath, @"host.json"), It.IsAny <FileMode>(), It.IsAny <FileAccess>(), It.IsAny <FileShare>())).Returns(testHostJsonStream);
fileSystem.SetupGet(f => f.Directory).Returns(dirBase.Object);
dirBase.Setup(d => d.EnumerateDirectories(rootPath))
.Returns(new[]
{
Path.Combine(rootPath, "function1"),
Path.Combine(rootPath, "function2"),
Path.Combine(rootPath, "function3")
});
var function1 = @"{
""scriptFile"": ""main.py"",
""disabled"": false,
""bindings"": [
{
""authLevel"": ""anonymous"",
""type"": ""httpTrigger"",
""direction"": ""in"",
""name"": ""req""
},
{
""type"": ""http"",
""direction"": ""out"",
""name"": ""$return""
}
]
}";
var function2 = @"{
""disabled"": false,
""scriptFile"": ""main.js"",
""bindings"": [
{
""name"": ""myQueueItem"",
""type"": ""orchestrationTrigger"",
""direction"": ""in"",
""queueName"": ""myqueue-items"",
""connection"": """"
}
]
}";
var function3 = @"{
""disabled"": false,
""scriptFile"": ""main.js"",
""bindings"": [
{
""name"": ""myQueueItem"",
""type"": ""activityTrigger"",
""direction"": ""in"",
""queueName"": ""myqueue-items"",
""connection"": """"
}
]
}";
fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function1\function.json"))).Returns(true);
fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function1\main.py"))).Returns(true);
fileBase.Setup(f => f.ReadAllText(Path.Combine(rootPath, @"function1\function.json"))).Returns(function1);
fileBase.Setup(f => f.Open(Path.Combine(rootPath, @"function1\function.json"), It.IsAny <FileMode>(), It.IsAny <FileAccess>(), It.IsAny <FileShare>())).Returns(() =>
{
return(new MemoryStream(Encoding.UTF8.GetBytes(function1)));
});
fileBase.Setup(f => f.Open(Path.Combine(testDataPath, "function1.dat"), It.IsAny <FileMode>(), It.IsAny <FileAccess>(), It.IsAny <FileShare>())).Returns(() =>
{
return(new MemoryStream(Encoding.UTF8.GetBytes(function1)));
});
fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function2\function.json"))).Returns(true);
fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function2\main.js"))).Returns(true);
fileBase.Setup(f => f.ReadAllText(Path.Combine(rootPath, @"function2\function.json"))).Returns(function2);
fileBase.Setup(f => f.Open(Path.Combine(rootPath, @"function2\function.json"), It.IsAny <FileMode>(), It.IsAny <FileAccess>(), It.IsAny <FileShare>())).Returns(() =>
{
return(new MemoryStream(Encoding.UTF8.GetBytes(function2)));
});
fileBase.Setup(f => f.Open(Path.Combine(testDataPath, "function2.dat"), It.IsAny <FileMode>(), It.IsAny <FileAccess>(), It.IsAny <FileShare>())).Returns(() =>
{
return(new MemoryStream(Encoding.UTF8.GetBytes(function1)));
});
fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function3\function.json"))).Returns(true);
fileBase.Setup(f => f.Exists(Path.Combine(rootPath, @"function3\main.js"))).Returns(true);
fileBase.Setup(f => f.ReadAllText(Path.Combine(rootPath, @"function3\function.json"))).Returns(function3);
fileBase.Setup(f => f.Open(Path.Combine(rootPath, @"function3\function.json"), It.IsAny <FileMode>(), It.IsAny <FileAccess>(), It.IsAny <FileShare>())).Returns(() =>
{
return(new MemoryStream(Encoding.UTF8.GetBytes(function3)));
});
fileBase.Setup(f => f.Open(Path.Combine(testDataPath, "function3.dat"), It.IsAny <FileMode>(), It.IsAny <FileAccess>(), It.IsAny <FileShare>())).Returns(() =>
{
return(new MemoryStream(Encoding.UTF8.GetBytes(function1)));
});
return(fileSystem.Object);
}
public void Create() => FileSystem.CreateDirectory(FullPath);
/// <exception cref="System.IO.IOException"/>
internal static void TestNameNodeRecoveryImpl(TestNameNodeRecovery.Corruptor corruptor
, bool finalize)
{
string TestPath = "/test/path/dir";
string TestPath2 = "/second/dir";
bool needRecovery = corruptor.NeedRecovery(finalize);
// start a cluster
Configuration conf = new HdfsConfiguration();
SetupRecoveryTestConf(conf);
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
Storage.StorageDirectory sd = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false
).Build();
cluster.WaitActive();
if (!finalize)
{
// Normally, the in-progress edit log would be finalized by
// FSEditLog#endCurrentLogSegment. For testing purposes, we
// disable that here.
FSEditLog spyLog = Org.Mockito.Mockito.Spy(cluster.GetNameNode().GetFSImage().GetEditLog
());
Org.Mockito.Mockito.DoNothing().When(spyLog).EndCurrentLogSegment(true);
DFSTestUtil.SetEditLogForTesting(cluster.GetNamesystem(), spyLog);
}
fileSys = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
FSImage fsimage = namesystem.GetFSImage();
fileSys.Mkdirs(new Path(TestPath));
fileSys.Mkdirs(new Path(TestPath2));
sd = fsimage.GetStorage().DirIterator(NNStorage.NameNodeDirType.Edits).Next();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
FilePath editFile = FSImageTestUtil.FindLatestEditsLog(sd).GetFile();
NUnit.Framework.Assert.IsTrue("Should exist: " + editFile, editFile.Exists());
// Corrupt the edit log
Log.Info("corrupting edit log file '" + editFile + "'");
corruptor.Corrupt(editFile);
// If needRecovery == true, make sure that we can't start the
// cluster normally before recovery
cluster = null;
try
{
Log.Debug("trying to start normally (this should fail)...");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).EnableManagedDfsDirsRedundancy
(false).Format(false).Build();
cluster.WaitActive();
cluster.Shutdown();
if (needRecovery)
{
NUnit.Framework.Assert.Fail("expected the corrupted edit log to prevent normal startup"
);
}
}
catch (IOException e)
{
if (!needRecovery)
{
Log.Error("Got unexpected failure with " + corruptor.GetName() + corruptor, e);
NUnit.Framework.Assert.Fail("got unexpected exception " + e.Message);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
// Perform NameNode recovery.
// Even if there was nothing wrong previously (needRecovery == false),
// this should still work fine.
cluster = null;
try
{
Log.Debug("running recovery...");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).EnableManagedDfsDirsRedundancy
(false).Format(false).StartupOption(recoverStartOpt).Build();
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("caught IOException while trying to recover. " + "message was "
+ e.Message + "\nstack trace\n" + StringUtils.StringifyException(e));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
// Make sure that we can start the cluster normally after recovery
cluster = null;
try
{
Log.Debug("starting cluster normally after recovery...");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).EnableManagedDfsDirsRedundancy
(false).Format(false).Build();
Log.Debug("successfully recovered the " + corruptor.GetName() + " corrupted edit log"
);
cluster.WaitActive();
NUnit.Framework.Assert.IsTrue(cluster.GetFileSystem().Exists(new Path(TestPath)));
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("failed to recover. Error message: " + e.Message);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public override void Delete() => FileSystem.RemoveDirectory(FullPath, recursive: false);
/// <summary>
/// Compile the views and copy compiled files to web project deploy (staging) folder.
/// </summary>
/// <param name="pathWebProj">Web Project path</param>
/// <param name="menuFileName">Menu file name</param>
/// <param name="pathFramework">Framework path</param>
private static bool StageFiles(string pathWebProj, string menuFileName, string pathFramework)
{
var pathDeploy = Path.Combine(pathWebProj, "Deploy");
var pathSource = Path.Combine(pathDeploy, "Source");
var pathBuild = Path.Combine(pathDeploy, "Build");
var pathBinFrom = Path.Combine(pathWebProj, "Bin");
var pathBinTo = Path.Combine(pathSource, "Bin");
var pathAreas = Path.Combine(pathSource, "Areas");
var moduleId = menuFileName.Substring(0, 2);
var pathAreaViewsFrom = Path.Combine(pathWebProj, @"Areas\" + moduleId + @"\Views");
var pathAreaViewsTo = Path.Combine(pathSource, @"Areas\" + moduleId + @"\Views");
// prepare compiled directories and files
Directory.CreateDirectory(pathBuild);
Directory.CreateDirectory(pathSource);
Directory.CreateDirectory(pathBinTo);
Directory.CreateDirectory(pathAreas);
Directory.CreateDirectory(pathAreaViewsTo);
File.Copy(Path.Combine(pathWebProj, "Web.config"), Path.Combine(pathSource, "Web.config"));
FileSystem.CopyDirectory(pathAreaViewsFrom, pathAreaViewsTo);
string[] patterns = { "System.*.dll", "Sage.CA.SBS.ERP.*.dll", "*.Web.Infrastructure.dll", "*.Web.dll", "*." + moduleId + ".*.dll" };
foreach (var pattern in patterns)
{
CopyFiles(pathBinFrom, pattern, pathBinTo, true);
}
// compile the razors views
var p = new Process
{
StartInfo =
{
FileName = Path.Combine(pathFramework, "aspnet_compiler.exe"),
Arguments = string.Format(" -v / -p \"{0}\" -fixednames \"{1}\"", pathSource, pathBuild)
}
};
p.Start();
p.WaitForExit();
#if _maybe_later
// CONSIDER FOR A FUTURE UPDATE.
// THE IDEA IS TO RETASK THE DEPLOY FOLDER AS A STAGING FOLDER THAT CONTAINS ALL FILES NEEDED
// IN ORDER TO DEPLOY A PROJECT (VIA COPY).
// AT PRESENT THE WEB PROJECT DEPLOY FOLDER DOES NOT CONTAIN ALL REQUIRED FILES.
// IT ALSO CONTAINS EXTRA FILES THAT SHOULD NOT BE DEPLOYED WITH AN ISV PROJECT.
// Copy bootstrapper file
var bootstrapFileName = moduleId + "bootstrapper.xml";
File.Copy(Path.Combine(pathWebProj, bootstrapFileName), Path.Combine(pathBuild, bootstrapFileName), true);
// Copy menu file to App_Data\MenuDetail
Directory.CreateDirectory(Path.Combine(pathBuild, "App_Data"));
Directory.CreateDirectory(Path.Combine(pathBuild, @"App_Data\MenuDetail"));
File.Copy(Path.Combine(pathWebProj, menuFileName), Path.Combine(pathBuild, @"App_Data\MenuDetail", menuFileName), true);
// Copy the scripts folder
var pathAreaScriptsFrom = Path.Combine(pathWebProj, @"Areas\" + moduleId + @"\Scripts");
var pathAreaScriptsTo = Path.Combine(pathBuild, @"Areas\" + moduleId + @"\Scripts");
Directory.CreateDirectory(pathAreaScriptsTo);
FileSystem.CopyDirectory(pathAreaScriptsFrom, pathAreaScriptsTo);
#endif
return(true);
}
private void OnShowInExplorerClicked(ContextMenuButton button)
{
var node = (ItemNode)button.ParentContextMenu.Tag;
FileSystem.ShowFileExplorer(Path.GetDirectoryName(node.Entry.SourceUrl));
}
/// <exception cref="System.IO.IOException"/>
protected internal override FileSystem GetLocalFS()
{
return(FileSystem.GetLocal(GetConf()).GetRawFileSystem());
}
/// <summary>
/// Copy bootStrapper, menuDetails and scripts files to the Sage Online folder
/// </summary>
/// <param name="pathWebProj">Web Project path</param>
/// <param name="pathSage300">Sage 300 folder</param>
/// <param name="menuFileName">Menu file name</param>
/// <returns>True if successful otherwise false</returns>
private static bool DeployFiles(string pathWebProj, string pathSage300, string menuFileName)
{
const string searchPattern = "*bootstrapper.xml";
var pathSageOnline = Path.Combine(pathSage300, @"Online");
var pathOnlineWeb = Path.Combine(pathSageOnline, "Web");
var pathOnlineWorker = Path.Combine(pathSageOnline, "Worker");
if (!Directory.Exists(pathOnlineWeb))
{
WriteErrorFile(pathWebProj, "The post-build utility MergeISVProject could not find the Online Web folder for the Web UIs. While the build was successful, the deployment was unsuccessful. Therefore, check view(s) for issue(s) (i.e. localization syntax).");
// Unsuccessful
return(false);
}
// Copy bootstrapper.xml file
var bootFiles = Directory.GetFiles(pathWebProj, searchPattern);
foreach (var srcfile in bootFiles)
{
var fileName = Path.GetFileName(srcfile);
if (fileName == null)
{
continue;
}
var desFile = Path.Combine(pathOnlineWeb, fileName);
File.Copy(srcfile, desFile, true);
desFile = Path.Combine(pathOnlineWorker, fileName);
File.Copy(srcfile, desFile, true);
}
// Copy menu file to App_Data menuDetails and all sub directory
var pathMenuFrom = Path.Combine(pathWebProj, menuFileName);
var pathMenuDir = Path.Combine(pathOnlineWeb, "App_Data", "MenuDetail");
var pathMenuTo = Path.Combine(pathMenuDir, menuFileName);
if (Directory.Exists(pathMenuDir))
{
File.Copy(pathMenuFrom, pathMenuTo, true);
foreach (var dir in Directory.GetDirectories(pathMenuDir))
{
var pathMenuSubTo = Path.Combine(dir, menuFileName);
File.Copy(pathMenuFrom, pathMenuSubTo, true);
}
}
// Copy areas scripts files
var pathAreaDir = Path.Combine(pathWebProj, "Areas");
if (Directory.Exists(pathAreaDir))
{
foreach (var dir in Directory.GetDirectories(pathAreaDir))
{
if (!dir.EndsWith("Core") && !dir.EndsWith("Shared"))
{
var paths = dir.Split('\\');
var pathSubArea = Path.Combine(pathOnlineWeb, "Areas", paths[paths.Length - 1]);
var pathFromScripts = Path.Combine(dir, "Scripts");
var pathToScripts = Path.Combine(pathSubArea, "Scripts");
FileSystem.CopyDirectory(pathFromScripts, pathToScripts, true);
}
}
}
//copy compiled files from deploy folder to sage online web area and bin directory
var pathDeploy = Path.Combine(pathWebProj, "Deploy");
var pathSource = Path.Combine(pathDeploy, "Source");
var pathBuild = Path.Combine(pathDeploy, "Build");
var moduleId = menuFileName.Substring(0, 2);
var pathBuildView = Path.Combine(pathBuild, @"Areas\" + moduleId + @"\Views");
var pathSageView = Path.Combine(pathSageOnline, @"Web\Areas\" + moduleId + @"\Views");
// Do not copy IF compile was not successful (determined by existance of folder)
if (!Directory.Exists(pathBuildView))
{
WriteErrorFile(pathWebProj, "The post-build utility MergeISVProject could not compile the razor view(s). While the build was successful, the deployment was unsuccessful. Therefore, check view(s) for issue(s) (i.e. localization syntax).");
// Unsuccessful
return(false);
}
FileSystem.CopyDirectory(pathBuildView, pathSageView, true);
var pathBuildBin = Path.Combine(pathBuild, "bin");
var pathSageBin = Path.Combine(pathOnlineWeb, "bin");
string[] ps = { "*.compiled", "App_Web_*.dll", "*.Web.dll", "*." + moduleId + ".*.dll" };
foreach (var pattern in ps)
{
CopyFiles(pathBuildBin, pattern, pathSageBin, true);
}
string[] psWorker = { "*." + moduleId + ".*.dll" };
foreach (var pattern in psWorker)
{
CopyFiles(pathBuildBin, pattern, pathOnlineWorker, true, false);
}
// copy resource satellite dlls for localization
CopyResourceSatelliteFiles(pathWebProj, pathSage300, moduleId);
return(true);
}
/// <exception cref="System.IO.IOException"/>
private void CleanupFile(FileSystem fileSys, Path name)
{
NUnit.Framework.Assert.IsTrue(fileSys.Exists(name));
fileSys.Delete(name, true);
NUnit.Framework.Assert.IsTrue(!fileSys.Exists(name));
}
static JProperty AssessGppNTService(JToken gppNtService)
{
int interestLevel = 1;
string gppNtServiceUid = Utility.GetSafeString(gppNtService, "@uid");
string gppNtServiceChanged = Utility.GetSafeString(gppNtService, "@changed");
string gppNtServiceName = Utility.GetSafeString(gppNtService, "@name");
string gppNtServiceUserName = Utility.GetSafeString(gppNtService["Properties"], "@accountName");
string gppNtServicecPassword = Utility.GetSafeString(gppNtService["Properties"], "@cpassword");
string gppNtServicePassword = "";
if (gppNtServicecPassword.Length > 0)
{
gppNtServicePassword = Utility.DecryptCpassword(gppNtServicecPassword);
interestLevel = 10;
}
string gppNtServiceTimeout = Utility.GetSafeString(gppNtService["Properties"], "@timeout");
string gppNtServiceStartup = Utility.GetSafeString(gppNtService["Properties"], "@startupType");
string gppNtServiceAction = Utility.GetSafeString(gppNtService["Properties"], "@serviceAction");
string gppNtServiceServiceName = Utility.GetSafeString(gppNtService["Properties"], "@serviceName");
string gppNtServiceFirstFailure = Utility.GetSafeString(gppNtService["Properties"], "@firstFailure");
string gppNtServiceSecondFailure = Utility.GetSafeString(gppNtService["Properties"], "@secondFailure");
string gppNtServiceThirdFailure = Utility.GetSafeString(gppNtService["Properties"], "@thirdFailure");
JObject gppNtServiceProgram =
FileSystem.InvestigatePath(Utility.GetSafeString(gppNtService["Properties"], "@program"));
JObject gppNtServiceArgs =
Utility.InvestigateString(Utility.GetSafeString(gppNtService["Properties"], "@args"));
if (gppNtServiceProgram["InterestLevel"] != null)
{
int progInterestLevelInt = int.Parse(gppNtServiceProgram["InterestLevel"].ToString());
if (progInterestLevelInt > interestLevel)
{
interestLevel = progInterestLevelInt;
}
}
if (gppNtServiceArgs["InterestLevel"] != null)
{
int argsInterestLevelInt = int.Parse(gppNtServiceArgs["InterestLevel"].ToString());
if (argsInterestLevelInt > interestLevel)
{
interestLevel = argsInterestLevelInt;
}
}
if (interestLevel >= GlobalVar.IntLevelToShow)
{
JObject assessedGppNtService = new JObject
{
{ "Name", gppNtServiceName }, { "Changed", gppNtServiceChanged }
};
if (gppNtServiceServiceName.Length > 0)
{
assessedGppNtService.Add("Service Name", gppNtServiceName);
}
if (gppNtServicecPassword.Length > 0)
{
assessedGppNtService.Add("Username", gppNtServiceUserName);
assessedGppNtService.Add("cPassword", gppNtServicecPassword);
assessedGppNtService.Add("Decrypted Password", gppNtServicePassword);
}
if (gppNtServiceTimeout.Length > 0)
{
assessedGppNtService.Add("Timeout", gppNtServiceTimeout);
}
if (gppNtServiceStartup.Length > 0)
{
assessedGppNtService.Add("Startup Type", gppNtServiceStartup);
}
if (gppNtServiceAction.Length > 0)
{
assessedGppNtService.Add("Action", gppNtServiceAction);
}
if (gppNtServiceFirstFailure.Length > 0)
{
assessedGppNtService.Add("Action on first failure", gppNtServiceFirstFailure);
}
if (gppNtServiceSecondFailure.Length > 0)
{
assessedGppNtService.Add("Action on second failure", gppNtServiceSecondFailure);
}
if (gppNtServiceThirdFailure.Length > 0)
{
assessedGppNtService.Add("Action on third failure", gppNtServiceThirdFailure);
}
if (gppNtServiceProgram.HasValues)
{
assessedGppNtService.Add("Program", gppNtServiceProgram);
}
if (gppNtServiceArgs.HasValues)
{
assessedGppNtService.Add("Args", gppNtServiceArgs);
}
return(new JProperty(gppNtServiceUid, assessedGppNtService));
}
return(null);
}
/// <summary>
/// Static method that sets the <code>FileSystem</code> to release back to
/// the
/// <see cref="Org.Apache.Hadoop.Lib.Service.FileSystemAccess"/>
/// service on servlet request completion.
/// </summary>
/// <param name="fs">fileystem instance.</param>
public static void SetFileSystem(FileSystem fs)
{
FileSystemTl.Set(fs);
}
public assembly_resolving()
{
given_project_repository(new FolderRepository(FileSystem.GetTempDirectory().GetDirectory(Guid.NewGuid().ToString()).MustExist(), FolderRepositoryOptions.SupportLocks));
TempDirectory = FileSystem.CreateTempDirectory();
}
public static string GetContentFolderForPackage(string packageFolder)
{
return(FileSystem.Combine(packageFolder, BottleFiles.WebContentFolder));
}
public async Task TestFullDeployment()
{
var sources = new Dictionary <string, string>()
{
{ @"Env\RootFile.json", "{ 'key1': 1, 'key2': 2 }" },
{ @"Env\Subfolder\Hello.txt", "Hello world" },
{ @"Env\Foo.txt", "Baz" },
{ @"Files\Foo.txt", "Bar" },
};
Dictionary <string, string> getSourceDrop(string root, string prefix)
{
return(sources.Where(e => e.Key.StartsWith(root))
.ToDictionary(e => e.Key.Substring(prefix.Length), e => e.Value));
}
var expectedSourceDrops = new Dictionary <string, Dictionary <string, string> >()
{
{
"file://Env", getSourceDrop(@"Env\", @"Env\")
},
{
"file://Files/Foo.txt", getSourceDrop(@"Files\Foo.txt", @"Files\")
},
{
"file://Env/Foo.txt", getSourceDrop(@"Env\Foo.txt", @"Env\")
}
};
var drops = new Dictionary <string, Dictionary <string, string> >()
{
{
"https://dev.azure.com/buildxlcachetest/drop/drops/dev/testdrop1?root=release/win-x64",
new Dictionary <string, string>
{
{ @"file1.bin", "File content 1" },
{ @"file2.txt", "File content 2" },
{ @"sub\file3.dll", "File content 3" }
}
},
{
"https://dev.azure.com/buildxlcachetest/drop/drops/dev/testdrop2?root=debug",
new Dictionary <string, string>
{
{ @"file1.bin", "File content 1" },
{ @"file2.txt", "File content 2 changed" },
{ @"sub\file5.dll", "File content 5" }
}
}
};
var config = new TestDeploymentConfig()
{
Drops =
{
new TestDeploymentConfig.DropDeploymentConfiguration()
{
{ "Url [Ring:Ring_0]", "https://dev.azure.com/buildxlcachetest/drop/drops/dev/testdrop1?root=release/win-x64" },
{ "Url [Ring:Ring_1]", "https://dev.azure.com/buildxlcachetest/drop/drops/dev/testdrop2?root=debug" },
},
new TestDeploymentConfig.DropDeploymentConfiguration()
{
{ "Url", "file://Env" },
},
new TestDeploymentConfig.DropDeploymentConfiguration()
{
{ "Url [Stamp:DefaultStamp]", "file://Files/Foo.txt" },
},
new TestDeploymentConfig.DropDeploymentConfiguration()
{
{ "Url [Stamp:DefaultStamp]", "file://Env/Foo.txt" },
}
}
};
var deploymentRunner = new DeploymentRunner(
Context,
sourceRoot: TestRootDirectoryPath / "src",
deploymentRoot: TestRootDirectoryPath / "deploy",
deploymentConfigurationPath: TestRootDirectoryPath / "DeploymentConfiguration.json",
FileSystem,
dropExeFilePath: TestRootDirectoryPath / @"dropbin\drop.exe",
retentionSizeGb: 1,
dropToken: DropToken);
// Write source files
WriteFiles(deploymentRunner.SourceRoot, sources);
var configText = JsonSerializer.Serialize(config, new JsonSerializerOptions()
{
WriteIndented = true
});
FileSystem.WriteAllText(deploymentRunner.DeploymentConfigurationPath, configText);
deploymentRunner.OverrideLaunchDropProcess = t =>
{
var dropContents = drops[t.dropUrl];
WriteFiles(new AbsolutePath(t.targetDirectory) / (t.relativeRoot ?? ""), dropContents);
return(BoolResult.Success);
};
await deploymentRunner.RunAsync().ShouldBeSuccess();
var manifestText = FileSystem.ReadAllText(deploymentRunner.DeploymentManifestPath);
var deploymentManifest = JsonSerializer.Deserialize <DeploymentManifest>(manifestText);
foreach (var drop in deploymentManifest.Drops)
{
var uri = new Uri(drop.Key);
var expectedDropContents = uri.IsFile ? expectedSourceDrops[drop.Key] : drops[drop.Key];
var layoutSpec = deploymentManifest.Drops[drop.Key];
layoutSpec.Count.Should().Be(expectedDropContents.Count);
foreach (var fileAndContent in expectedDropContents)
{
var hash = new ContentHash(layoutSpec[fileAndContent.Key].Hash);
var expectedPath = deploymentRunner.DeploymentRoot / DeploymentUtilities.GetContentRelativePath(hash);
var text = FileSystem.ReadAllText(expectedPath);
text.Should().Be(fileAndContent.Value);
}
}
}
private async Task PutFileAcrossDrivesAsync(
FileRealizationMode allowedFileRealizationMode,
bool contentAlreadyCached,
bool contentShouldBeCached,
Func<PutResult, bool> checkResult)
{
// This only works when we have multiple drives.
var memoryFileSystem = FileSystem as MemoryFileSystem;
if (memoryFileSystem == null)
{
return;
}
using (var testDirectory = new DisposableDirectory(FileSystem))
{
var context = new Context(Logger);
try
{
using (var store = Create(testDirectory.Path, Clock))
{
await store.StartupAsync(context).ShouldBeSuccess();
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
ContentHash contentHash = bytes.CalculateHash(ContentHashType);
// Verify content doesn't exist yet in store
Assert.False(await store.ContainsAsync(context, contentHash, null));
var pathToContentDifferentVolume = new AbsolutePath(PathGeneratorUtilities.GetAbsolutePath("D", "foo.txt"));
try
{
FileSystem.WriteAllBytes(pathToContentDifferentVolume, bytes);
if (contentAlreadyCached)
{
await store.PutFileAsync(
context, pathToContentDifferentVolume, FileRealizationMode.Copy, ContentHashType, null).ShouldBeSuccess();
}
var result = await store.PutFileAsync(
context, pathToContentDifferentVolume, allowedFileRealizationMode, ContentHashType, null);
Assert.True(checkResult(result));
(await store.ContainsAsync(context, contentHash, null)).Should()
.Be(contentShouldBeCached);
}
finally
{
FileSystem.DeleteFile(pathToContentDifferentVolume);
}
await store.ShutdownAsync(context).ShouldBeSuccess();
}
}
finally
{
FileSystem.DeleteDirectory(testDirectory.Path, DeleteOptions.All);
}
}
}
/// <summary>
/// Starts the updater executable and sends update data to it
/// </summary>
/// <param name="relaunchApplication">true if relaunching the caller application is required; false otherwise</param>
/// <param name="updaterDoLogging">true if the updater writes to a log file; false otherwise</param>
/// <param name="updaterShowConsole">true if the updater shows the console window; false otherwise</param>
/// <returns>True if successful (unless a restart was required</returns>
public void ApplyUpdates(bool relaunchApplication, bool updaterDoLogging, bool updaterShowConsole)
{
if (IsWorking)
{
throw new InvalidOperationException("Another update process is already in progress");
}
lock (UpdatesToApply)
{
using (WorkScope.New(isWorking => IsWorking = isWorking))
{
bool revertToDefaultBackupPath = true;
// Set current directory the the application directory
// this prevents the updater from writing to e.g. c:\windows\system32
// if the process is started by autorun on windows logon.
// ReSharper disable AssignNullToNotNullAttribute
Environment.CurrentDirectory = Path.GetDirectoryName(ApplicationPath);
// ReSharper restore AssignNullToNotNullAttribute
// Make sure the current backup folder is accessible for writing from this process
string backupParentPath = Path.GetDirectoryName(Config.BackupFolder) ?? string.Empty;
if (Directory.Exists(backupParentPath) && PermissionsCheck.HaveWritePermissionsForFolder(backupParentPath))
{
// Remove old backup folder, in case this same folder was used previously,
// and it wasn't removed for some reason
try
{
if (Directory.Exists(Config.BackupFolder))
{
FileSystem.DeleteDirectory(Config.BackupFolder);
}
revertToDefaultBackupPath = false;
}
catch (UnauthorizedAccessException)
{
}
// Attempt to (re-)create the backup folder
try
{
Directory.CreateDirectory(Config.BackupFolder);
if (!PermissionsCheck.HaveWritePermissionsForFolder(Config.BackupFolder))
{
revertToDefaultBackupPath = true;
}
}
catch (UnauthorizedAccessException)
{
// We're having permissions issues with this folder, so we'll attempt
// using a backup in a default location
revertToDefaultBackupPath = true;
}
}
if (revertToDefaultBackupPath)
{
Config._backupFolder = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
Config.UpdateProcessName + "UpdateBackups" + DateTime.UtcNow.Ticks);
try
{
Directory.CreateDirectory(Config.BackupFolder);
}
catch (UnauthorizedAccessException ex)
{
// We can't backup, so we abort
throw new UpdateProcessFailedException("Could not create backup folder " + Config.BackupFolder, ex);
}
}
bool runPrivileged = false, hasColdUpdates = false;
State = UpdateProcessState.RollbackRequired;
foreach (var task in UpdatesToApply)
{
IUpdateTask t = task;
task.ProgressDelegate += status => TaskProgressCallback(status, t);
try
{
// Execute the task
task.ExecutionStatus = task.Execute(false);
}
catch (Exception ex)
{
task.ExecutionStatus = TaskExecutionStatus.Failed; // mark the failing task before rethrowing
throw new UpdateProcessFailedException("Update task execution failed: " + task, ex);
}
if (task.ExecutionStatus == TaskExecutionStatus.RequiresAppRestart ||
task.ExecutionStatus == TaskExecutionStatus.RequiresPrivilegedAppRestart)
{
// Record that we have cold updates to run, and if required to run any of them privileged
runPrivileged = runPrivileged || task.ExecutionStatus == TaskExecutionStatus.RequiresPrivilegedAppRestart;
hasColdUpdates = true;
continue;
}
// We are being quite explicit here - only Successful return values are considered
// to be Ok (cold updates are already handled above)
if (task.ExecutionStatus != TaskExecutionStatus.Successful)
{
throw new UpdateProcessFailedException("Update task execution failed: " + task);
}
}
// If an application restart is required
if (hasColdUpdates)
{
var dto = new NauIpc.NauDto
{
Configs = Instance.Config,
Tasks = Instance.UpdatesToApply,
AppPath = ApplicationPath,
WorkingDirectory = Environment.CurrentDirectory,
RelaunchApplication = relaunchApplication,
LogItems = Logger.LogItems,
};
NauIpc.ExtractUpdaterFromResource(Config.TempFolder, Instance.Config.UpdateExecutableName);
var info = new ProcessStartInfo
{
UseShellExecute = true,
WorkingDirectory = Environment.CurrentDirectory,
FileName = Path.Combine(Config.TempFolder, Instance.Config.UpdateExecutableName),
Arguments =
string.Format(@"""{0}"" {1} {2}", Config.UpdateProcessName,
updaterShowConsole ? "-showConsole" : string.Empty,
updaterDoLogging ? "-log" : string.Empty),
};
if (!updaterShowConsole)
{
info.WindowStyle = ProcessWindowStyle.Hidden;
info.CreateNoWindow = true;
}
// If we can't write to the destination folder, then lets try elevating priviledges.
if (runPrivileged || !PermissionsCheck.HaveWritePermissionsForFolder(Environment.CurrentDirectory))
{
info.Verb = "runas";
}
bool createdNew;
_shutdownMutex = new Mutex(true, Config.UpdateProcessName + "Mutex", out createdNew);
try
{
NauIpc.LaunchProcessAndSendDto(dto, info, Config.UpdateProcessName);
}
catch (Exception ex)
{
throw new UpdateProcessFailedException("Could not launch cold update process", ex);
}
Environment.Exit(0);
}
State = UpdateProcessState.AppliedSuccessfully;
UpdatesToApply.Clear();
}
}
}
void ProcessSection(Section section, ContainerClass sections, string datapath = null, FileSystem vfs = null)
{
var tgt = sections.GetSection(section.Name);
if (tgt == null)
{
FLLog.Warning("Ini", "Unknown section " + section.Name + FormatLine(section.File, section.Line));
return;
}
if (tgt.Field == null)
{
GetFromSection(section, tgt.Type, this);
}
else
{
if (tgt.Delimiters != null)
{
foreach (var ch in Chunk(tgt.Delimiters, section))
{
var childObject = GetFromSection(ch, tgt.Type, null, datapath, vfs);
if (childObject != null)
{
var list = (IList)tgt.Field.GetValue(this);
list.Add(childObject);
}
}
}
else
{
var parsed = GetFromSection(section, tgt.Type, null, datapath, vfs);
if (parsed != null)
{
if (tgt.IsList)
{
var list = (IList)tgt.Field.GetValue(this);
if (tgt.AttachToParent)
{
var count = list.Count;
if (count <= 0)
{
FLLog.Warning("Ini", $"Section {section.Name} has no parent {FormatLine(section.File, section.Line)}");
return;
}
var parent = list[count - 1];
bool success = false;
var parentInfo = GetSectionInfo(parent.GetType());
foreach (var cs in parentInfo.ChildSections)
{
if (cs.Name.Equals(section.Name, StringComparison.OrdinalIgnoreCase))
{
var ls2 = (IList)cs.Field.GetValue(parent);
ls2.Add(parsed);
success = true;
break;
}
}
if (!success)
{
FLLog.Warning("Ini",
$"Type {parentInfo.GetType().Name} does not accept child section {section.Name} {FormatLine(section.File, section.Line)}");
}
}
else
{
list.Add(parsed);
}
}
else
{
tgt.Field.SetValue(this, parsed);
}
}
}
}
}
static void Main(string[] args)
{
PerfTracker.StartupEvent("Entered main");
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(UAlbion.Api.Event)));
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(UAlbion.Core.Events.HelpEvent)));
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(UAlbion.Core.Veldrid.Events.InputEvent)));
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(UAlbion.Editor.EditorSetPropertyEvent)));
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(UAlbion.Formats.ScriptEvents.PartyMoveEvent)));
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(UAlbion.Game.Events.StartEvent)));
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(UAlbion.Game.Veldrid.Debugging.HideDebugWindowEvent)));
Event.AddEventsFromAssembly(Assembly.GetAssembly(typeof(IsoYawEvent)));
PerfTracker.StartupEvent("Built event parsers");
var commandLine = new CommandLineOptions(args);
if (commandLine.Mode == ExecutionMode.Exit)
{
return;
}
PerfTracker.StartupEvent($"Running as {commandLine.Mode}");
var disk = new FileSystem();
var jsonUtil = new FormatJsonUtil();
var baseDir = ConfigUtil.FindBasePath(disk);
if (baseDir == null)
{
throw new InvalidOperationException("No base directory could be found.");
}
PerfTracker.StartupEvent($"Found base directory {baseDir}");
if (commandLine.Mode == ExecutionMode.ConvertAssets)
{
ConvertAssets.Convert(
disk,
jsonUtil,
commandLine.ConvertFrom,
commandLine.ConvertTo,
commandLine.DumpIds,
commandLine.DumpAssetTypes,
commandLine.ConvertFilePattern);
return;
}
var(exchange, services) = AssetSystem.SetupAsync(baseDir, disk, jsonUtil).Result;
if (commandLine.NeedsEngine)
{
BuildEngine(commandLine, exchange);
}
services.Add(new StdioConsoleReader());
var assets = exchange.Resolve <IAssetManager>();
AutodetectLanguage(exchange, assets);
switch (commandLine.Mode) // ConvertAssets handled above as it requires a specialised asset system setup
{
case ExecutionMode.Game: Albion.RunGame(exchange, services, baseDir, commandLine); break;
case ExecutionMode.BakeIsometric: IsometricTest.Run(exchange, commandLine); break;
case ExecutionMode.DumpData:
PerfTracker.BeginFrame(); // Don't need to show verbose startup logging while dumping
var tf = new TextFormatter();
exchange.Attach(tf);
var parsedIds = commandLine.DumpIds?.Select(AssetId.Parse).ToArray();
if ((commandLine.DumpFormats & DumpFormats.Json) != 0)
{
DumpJson.Dump(baseDir, assets, commandLine.DumpAssetTypes, parsedIds);
}
if ((commandLine.DumpFormats & DumpFormats.Text) != 0)
{
DumpText.Dump(assets, baseDir, tf, commandLine.DumpAssetTypes, parsedIds);
}
if ((commandLine.DumpFormats & DumpFormats.Png) != 0)
{
var dumper = new DumpGraphics();
exchange.Attach(dumper);
dumper.Dump(baseDir, commandLine.DumpAssetTypes, commandLine.DumpFormats, parsedIds);
}
//if ((commandLine.DumpFormats & DumpFormats.Tiled) != 0)
// DumpTiled.Dump(baseDir, assets, commandLine.DumpAssetTypes, parsedIds);
break;
case ExecutionMode.Exit: break;
}
Console.WriteLine("Exiting");
exchange.Dispose();
}
public Task PutSameContentManyTimesTest(bool useRedundantPutFileShortcut)
{
var context = new Context(Logger);
ContentStoreSettings = new ContentStoreSettings()
{
UseRedundantPutFileShortcut = useRedundantPutFileShortcut
};
return TestStore(context, Clock, async store =>
{
byte[] bytes = ThreadSafeRandom.GetBytes(ValueSize);
ContentHash contentHash = bytes.CalculateHash(ContentHashType);
// Verify content doesn't exist yet in store
Assert.False(await store.ContainsAsync(context, contentHash, null));
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
ContentHash hashFromPut;
using (var pinContext = store.CreatePinContext())
{
var concurrency = 24;
var iterations = 100;
var items = Enumerable.Range(0, concurrency).Select(i =>
{
AbsolutePath pathToContent = tempDirectory.Path / $"tempContent{i}.txt";
FileSystem.WriteAllBytes(pathToContent, bytes);
return (pathToContent, iterations);
}).ToArray();
int nonDuplicatedPuts = 0;
await ParallelAlgorithms.WhenDoneAsync(24, CancellationToken.None, async (scheduleItem, item) =>
{
// Put the content into the store w/ hard link
var r = await store.PutFileAsync(
context, item.pathToContent, FileRealizationMode.Any, ContentHashType, new PinRequest(pinContext));
hashFromPut = r.ContentHash;
if (!r.ContentAlreadyExistsInCache)
{
Interlocked.Increment(ref nonDuplicatedPuts);
}
Clock.Increment();
Assert.True(pinContext.Contains(hashFromPut));
if (item.iterations != 0)
{
scheduleItem((item.pathToContent, item.iterations - 1));
}
},
items);
Assert.Equal(1, nonDuplicatedPuts);
}
}
});
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
int numDataNodes = 0;
int numRacks = 0;
bool inject = false;
long startingBlockId = 1;
int numBlocksPerDNtoInject = 0;
int replication = 1;
bool checkDataNodeAddrConfig = false;
long simulatedCapacityPerDn = SimulatedFSDataset.DefaultCapacity;
string bpid = null;
Configuration conf = new HdfsConfiguration();
for (int i = 0; i < args.Length; i++)
{
// parse command line
if (args[i].Equals("-n"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("missing number of nodes");
}
numDataNodes = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-racks"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing number of racks");
}
numRacks = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-r"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing replication factor");
}
replication = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-d"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing datanode dirs parameter");
}
dataNodeDirs = args[i];
}
else
{
if (args[i].Equals("-simulated"))
{
SimulatedFSDataset.SetFactory(conf);
if ((i + 1) < args.Length && !args[i + 1].StartsWith("-"))
{
simulatedCapacityPerDn = long.Parse(args[++i]);
}
}
else
{
if (args[i].Equals("-bpid"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing blockpoolid parameter");
}
bpid = args[i];
}
else
{
if (args[i].Equals("-inject"))
{
if (!FsDatasetSpi.Factory.GetFactory(conf).IsSimulated())
{
System.Console.Out.Write("-inject is valid only for simulated");
PrintUsageExit();
}
inject = true;
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing starting block and number of blocks per DN to inject");
}
startingBlockId = System.Convert.ToInt32(args[i]);
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing number of blocks to inject");
}
numBlocksPerDNtoInject = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-checkDataNodeAddrConfig"))
{
checkDataNodeAddrConfig = true;
}
else
{
PrintUsageExit();
}
}
}
}
}
}
}
}
}
if (numDataNodes <= 0 || replication <= 0)
{
PrintUsageExit("numDataNodes and replication have to be greater than zero");
}
if (replication > numDataNodes)
{
PrintUsageExit("Replication must be less than or equal to numDataNodes");
}
if (bpid == null)
{
PrintUsageExit("BlockPoolId must be provided");
}
string nameNodeAdr = FileSystem.GetDefaultUri(conf).GetAuthority();
if (nameNodeAdr == null)
{
System.Console.Out.WriteLine("No name node address and port in config");
System.Environment.Exit(-1);
}
bool simulated = FsDatasetSpi.Factory.GetFactory(conf).IsSimulated();
System.Console.Out.WriteLine("Starting " + numDataNodes + (simulated ? " Simulated "
: " ") + " Data Nodes that will connect to Name Node at " + nameNodeAdr);
Runtime.SetProperty("test.build.data", dataNodeDirs);
long[] simulatedCapacities = new long[numDataNodes];
for (int i_1 = 0; i_1 < numDataNodes; ++i_1)
{
simulatedCapacities[i_1] = simulatedCapacityPerDn;
}
MiniDFSCluster mc = new MiniDFSCluster();
try
{
mc.FormatDataNodeDirs();
}
catch (IOException e)
{
System.Console.Out.WriteLine("Error formating data node dirs:" + e);
}
string[] rack4DataNode = null;
if (numRacks > 0)
{
System.Console.Out.WriteLine("Using " + numRacks + " racks: ");
string rackPrefix = GetUniqueRackPrefix();
rack4DataNode = new string[numDataNodes];
for (int i_2 = 0; i_2 < numDataNodes; ++i_2)
{
//rack4DataNode[i] = racks[i%numRacks];
rack4DataNode[i_2] = rackPrefix + "-" + i_2 % numRacks;
System.Console.Out.WriteLine("Data Node " + i_2 + " using " + rack4DataNode[i_2]);
}
}
try
{
mc.StartDataNodes(conf, numDataNodes, true, HdfsServerConstants.StartupOption.Regular
, rack4DataNode, null, simulatedCapacities, false, checkDataNodeAddrConfig);
Sharpen.Thread.Sleep(10 * 1000);
// Give the DN some time to connect to NN and init storage directories.
if (inject)
{
long blockSize = 10;
System.Console.Out.WriteLine("Injecting " + numBlocksPerDNtoInject + " blocks in each DN starting at blockId "
+ startingBlockId + " with blocksize of " + blockSize);
Block[] blocks = new Block[numBlocksPerDNtoInject];
long blkid = startingBlockId;
for (int i_dn = 0; i_dn < numDataNodes; ++i_dn)
{
for (int i_2 = 0; i_2 < blocks.Length; ++i_2)
{
blocks[i_2] = new Block(blkid++, blockSize, CreateEditsLog.BlockGenerationStamp);
}
for (int i_3 = 1; i_3 <= replication; ++i_3)
{
// inject blocks for dn_i into dn_i and replica in dn_i's neighbors
mc.InjectBlocks((i_dn + i_3 - 1) % numDataNodes, Arrays.AsList(blocks), bpid);
System.Console.Out.WriteLine("Injecting blocks of dn " + i_dn + " into dn" + ((i_dn
+ i_3 - 1) % numDataNodes));
}
}
System.Console.Out.WriteLine("Created blocks from Bids " + startingBlockId + " to "
+ (blkid - 1));
}
}
catch (IOException e)
{
System.Console.Out.WriteLine("Error creating data node:" + e);
}
}
private void NewRecord_Click(System.Object sender, System.EventArgs e)
{
FileSystem.Reset();
}
static object GetFromSection(Section s, ReflectionInfo type, object obj = null, string datapath = null, FileSystem vfs = null)
{
if (obj == null)
{
obj = Activator.CreateInstance(type.Type);
}
ulong bitmask = 0;
ulong requiredBits = type.RequiredFields;
foreach (var e in s)
{
//Find entry
var eHash = Hash(e.Name);
int idx = -1;
for (int i = 0; i < type.Fields.Count; i++)
{
if (eHash == type.FieldHashes[i] &&
type.Fields[i].Attr.Name.Equals(e.Name, StringComparison.OrdinalIgnoreCase))
{
idx = i;
break;
}
}
//Special Handling: Use sparingly
if (idx == -1)
{
bool handled = false;
if (obj is ICustomEntryHandler ce)
{
foreach (var k in ce.CustomEntries)
{
if (k.Hash == eHash)
{
k.Handler(ce, e);
handled = true;
break;
}
}
}
else if (obj is IEntryHandler eh)
{
eh.HandleEntry(e);
handled = true;
}
if (!handled)
{
FLLog.Warning("Ini", "Unknown entry " + e.Name + FormatLine(e.File, e.Line, s.Name));
}
continue;
}
var field = type.Fields[idx];
//Warning for duplicates
if (!field.Attr.Multiline)
{
if ((bitmask & (1ul << idx)) != 0)
{
FLLog.Warning("Ini", "Duplicate of " + field.Attr.Name + FormatLine(e.File, e.Line, s.Name));
}
bitmask |= 1ul << idx;
}
requiredBits &= ~(1ul << idx);
//Handle by type
var ftype = field.Field.FieldType;
Type nType;
if (field.NullableType != null)
{
ftype = field.NullableType;
}
//Fill
if (ftype == typeof(string))
{
if (ComponentCheck(1, s, e))
{
field.Field.SetValue(obj, e[0].ToString());
}
}
else if (ftype == typeof(float))
{
if (ComponentCheck(1, s, e))
{
field.Field.SetValue(obj, e[0].ToSingle());
}
}
else if (ftype == typeof(int))
{
if (ComponentCheck(1, s, e))
{
field.Field.SetValue(obj, e[0].ToInt32());
}
}
else if (ftype == typeof(long))
{
if (ComponentCheck(1, s, e))
{
field.Field.SetValue(obj, e[0].ToInt64());
}
}
else if (ftype == typeof(bool))
{
if (field.Attr.Presence)
{
field.Field.SetValue(obj, true);
}
else if (ComponentCheck(1, s, e))
{
field.Field.SetValue(obj, e[0].ToBoolean());
}
}
else if (ftype == typeof(Vector3))
{
if (e.Count == 1 && e[0].ToSingle() == 0)
{
field.Field.SetValue(obj, Vector3.Zero);
}
else if (field.Attr.Mode == Vec3Mode.None)
{
if (ComponentCheck(3, s, e))
{
field.Field.SetValue(obj, new Vector3(e[0].ToSingle(), e[1].ToSingle(), e[2].ToSingle()));
}
}
else if (ComponentCheck(3, s, e, 1))
{
if (field.Attr.Mode == Vec3Mode.Size)
{
if (e.Count == 1)
{
field.Field.SetValue(obj, new Vector3(e[0].ToSingle()));
}
else if (e.Count == 2)
{
field.Field.SetValue(obj, new Vector3(e[0].ToSingle(), e[1].ToSingle(), 0));
}
else
{
field.Field.SetValue(obj, new Vector3(e[0].ToSingle(), e[1].ToSingle(), e[2].ToSingle()));
}
}
else
{
//optional components
var v3 = Vector3.Zero;
v3.X = e[0].ToSingle();
if (e.Count > 1)
{
v3.Y = e[1].ToSingle();
}
if (e.Count > 2)
{
v3.Z = e[2].ToSingle();
}
field.Field.SetValue(obj, v3);
}
}
}
else if (ftype == typeof(Quaternion))
{
if (ComponentCheck(4, s, e))
{
field.Field.SetValue(obj, new Quaternion(e[1].ToSingle(), e[2].ToSingle(), e[3].ToSingle(), e[0].ToSingle()));
}
}
else if (ftype == typeof(Vector4))
{
if (ComponentCheck(4, s, e))
{
field.Field.SetValue(obj, new Vector4(e[0].ToSingle(), e[1].ToSingle(), e[2].ToSingle(), e[3].ToSingle()));
}
}
else if (ftype == typeof(Vector2))
{
if (e.Count == 1 && field.Attr.MinMax)
{
field.Field.SetValue(obj, new Vector2(-1, e[0].ToSingle()));
}
else if (ComponentCheck(2, s, e))
{
field.Field.SetValue(obj, new Vector2(e[0].ToSingle(), e[1].ToSingle()));
}
}
else if (ftype == typeof(Color4))
{
if (ComponentCheck(4, s, e, 3))
{
Color4 col;
if (e.Count == 3)
{
col = new Color4(e[0].ToInt32() / 255f, e[1].ToInt32() / 255f, e[2].ToInt32() / 255f, 1f);
}
else
{
col = new Color4(e[0].ToInt32() / 255f, e[1].ToInt32() / 255f, e[2].ToInt32() / 255f, e[3].ToInt32() / 255f);
}
field.Field.SetValue(obj, col);
}
}
else if (ftype == typeof(Color3f))
{
if (ComponentCheck(3, s, e))
{
if (field.Attr.FloatColor)
{
field.Field.SetValue(obj, new Color3f(e[0].ToSingle(), e[1].ToSingle(), e[2].ToSingle()));
}
else
{
field.Field.SetValue(obj, new Color3f(e[0].ToSingle() / 255f, e[1].ToSingle() / 255f, e[2].ToSingle() / 255f));
}
}
}
else if (ftype == typeof(List <string>))
{
if (field.Attr.Multiline)
{
bitmask &= ~(1ul << idx); //Avoid duplicate warnings
if (ComponentCheck(1, s, e))
{
var v = (List <string>)field.Field.GetValue(obj);
v.Add(e[0].ToString());
}
}
else if (ComponentCheck(int.MaxValue, s, e, 1))
{
field.Field.SetValue(obj, e.Select((x) => x.ToString()).ToList());
}
}
else if (ftype == typeof(List <int>))
{
bitmask &= ~(1ul << idx); //Avoid duplicate warnings
if (ComponentCheck(1, s, e))
{
var v = (List <int>)field.Field.GetValue(obj);
if (v == null)
{
v = new List <int>();
field.Field.SetValue(obj, v);
}
v.Add(e[0].ToInt32());
}
}
else if (ftype == typeof(float[]))
{
if (ComponentCheck(int.MaxValue, s, e, 1))
{
var floats = new float[e.Count];
for (int i = 0; i < e.Count; i++)
{
floats[i] = e[i].ToSingle();
}
field.Field.SetValue(obj, floats);
}
}
else if (ftype == typeof(int[]))
{
if (ComponentCheck(int.MaxValue, s, e, 1))
{
var ints = new int[e.Count];
for (int i = 0; i < e.Count; i++)
{
ints[i] = e[i].ToInt32();
}
field.Field.SetValue(obj, ints);
}
}
else if (ftype == typeof(string[]))
{
if (ComponentCheck(int.MaxValue, s, e, 1))
{
var strings = new string[e.Count];
for (int i = 0; i < e.Count; i++)
{
strings[i] = e[i].ToString();
}
field.Field.SetValue(obj, strings);
}
}
else if (ftype == typeof(List <string[]>))
{
if (ComponentCheck(int.MaxValue, s, e, 1))
{
var strings = new string[e.Count];
for (int i = 0; i < e.Count; i++)
{
strings[i] = e[i].ToString();
}
var lst = (List <string[]>)field.Field.GetValue(obj);
lst.Add(strings);
}
}
else if (ftype == typeof(Guid))
{
if (ComponentCheck(1, s, e, 1))
{
if (Guid.TryParse(e[0].ToString(), out var g))
{
field.Field.SetValue(obj, g);
}
else
{
FLLog.Warning("Ini", "Unable to parse GUID" + FormatLine(e.File, e.Line, s.Name));
}
}
}
else if (ftype.IsEnum)
{
if (ComponentCheck(1, s, e))
{
//TryParse requires generics, wrap in exception handler
try
{
field.Field.SetValue(obj, Enum.Parse(ftype, e[0].ToString(), true));
}
catch (Exception)
{
FLLog.Error("Ini", "Invalid value for enum " + e[0].ToString() + FormatLine(e.File, e.Line, s.Name));
}
}
}
}
if (requiredBits != 0)
{
//These sections crash the game if they don't have required fields
//So don't let them be added to lists
for (int i = 0; i < 64; i++)
{
if ((requiredBits & (1ul << i)) != 0)
{
FLLog.Error("Ini", string.Format("Missing required field {0}{1}", type.Fields[i].Attr.Name, FormatLine(s.File, s.Line, s.Name)));
}
}
return(null);
}
else
{
if (datapath != null && obj is IniFile ini)
{
ini.OnSelfFilled(datapath, vfs);
}
return(obj);
}
}
