/// <summary>
/// Sorts files in the input path.
/// </summary>
/// <param name="inputPath">Either file name or directory name. In the latter case the directory will be processed recursively
/// and all files will be merged together in sorted order.</param>
/// <param name="tempDir">Directory for temporary files.</param>
/// <param name="inMemSortCount">Number of elements to sort in memory. Must be found impirically. Usually the more memory is
/// used the faster the sort. At the moment there is a .NET limitation of 2G of bytes.</param>
/// <param name="element">IElement.</param>
/// <param name="resultFile">Full name of the file to store the result.</param>
public void Sort(string inputPath, string tempDir, Int64 inMemSortCount, IElement element, string resultFile)
{
if (inMemSortCount > int.MaxValue)
{
throw new ArgumentOutOfRangeException(
String.Format("Memory sorting size > {0} is not supported yet", int.MaxValue));
}
DirectoryExt.Delete(tempDir);
Directory.CreateDirectory(tempDir);
DateTime start = DateTime.Now;
T[] arr = new T[inMemSortCount];
_chunkCount = 0;
Int64 elCount = SortChuncksPath(inputPath, tempDir, arr, element);
double time = (DateTime.Now - start).TotalSeconds;
if (IsVerbose)
{
Console.WriteLine("Sorted {0:#,#} chuncks, {1:#,#} elements, {2:#,#} el/s",
_chunkCount, elCount, elCount / time);
}
Merge(tempDir, element, resultFile);
}
/// <summary>
/// refresh view base on directory
/// </summary>
/// <param name="di"></param>
private void RefreshView(DirectoryInfo di)
{
// only ask if it is a dir change
if (this.di != null && di.FullName != this.di.FullName)
{
AskToCommit();
}
this.di = di;
FileInfo[] files = null;
try
{
//files = di.GetFiles( "*.mp3", searchOption );
//BKP Todo config types
files = DirectoryExt.GetFileInfos(di, "*.mp3;*.flac", searchOption);
if (files != null)
{
RefreshView(files);
}
}
catch (UnauthorizedAccessException)
{
return;
}
catch (IOException)
{
return;
}
}
/// <summary>
/// Writes out the path to the grep executable to the app data folder. Used in the VS plugin to find the exe.
/// </summary>
private void writePathToAppData()
{
String filePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), @"Barools\Grep\path.txt");
DirectoryExt.Create(Path.GetDirectoryName(filePath));
File.WriteAllText(filePath, System.Reflection.Assembly.GetExecutingAssembly().GetName().CodeBase.Substring(8));
}
/// <summary>
/// Creates a copy of a file, making sure the destination directory exists.
/// </summary>
/// <param name="src">The filepath to copy.</param>
/// <param name="dest">The filepath to make for the new file.</param>
/// <param name="overwrite">Set to true to overwrite any files that may already exist in
/// the destination directory. If false, and the destination file already exists, an error
/// is thrown.</param>
public static void Copy(string src, string dest, bool overwrite)
{
var destDir = Path.GetDirectoryName(dest);
DirectoryExt.CreateDirectory(destDir);
File.Copy(src, dest, overwrite);
}
/// <summary>
/// Writes the given text to a file, ensuring the parent directory for
/// the file actually exists.
/// </summary>
/// <param name="path">The filepath to write.</param>
/// <param name="text">The content to write to the file.</param>
public static void WriteAllText(string path, string text)
{
var dir = Path.GetDirectoryName(path);
DirectoryExt.CreateDirectory(dir);
File.WriteAllText(path, text);
}
public void CopyTest()
{
string sourcedir = AppDomain.CurrentDomain.BaseDirectory;
string targetdir = System.IO.Path.Combine(System.IO.Directory.GetParent(sourcedir).FullName, "target");
DirectoryExt.Copy(sourcedir, targetdir);
}
private void DownloadIfRequired()
{
if (!Url.EndsWith(".zip"))
{
throw new NotSupportedException("Not zip archives are not supported");
}
ExtractPath = DownloadPath;
RootPath = Path.Combine(ExtractPath, Name);
ArchiveName = RootPath + ".zip";
if (Rewrite || IsDirectoryNotExistsOrEmpty())
{
// Block another processes which try to use the same files.
using (var zipFileNameMutex = new Mutex(false, ConvertToValidMutexName(ArchiveName)))
{
if (!zipFileNameMutex.WaitOne())
{
return;
}
try
{
if (Rewrite || IsDirectoryNotExistsOrEmpty())
{
if (DirectoryExt.Exists(RootPath))
{
DirectoryExt.Delete(RootPath);
}
if (FileExt.Exists(ArchiveName))
{
if (Rewrite)
{
FileExt.Delete(ArchiveName);
}
DownloadArchive();
}
else
{
DownloadArchive();
}
}
else
{
Logger.LogInfo($"{Name} already downloaded and unpacked.");
}
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
RootPath = null;
Logger.LogError(ex);
}
finally
{
zipFileNameMutex.ReleaseMutex();
}
}
}
}
protected void Dump(string data, TextFile sourceFile, bool tokens)
{
DirectoryExt.CreateDirectory(DumpDir);
string name = string.IsNullOrEmpty(sourceFile.Name) ? "" : sourceFile.Name + ".";
FileExt.WriteAllText(Path.Combine(DumpDir, $"{name}{(tokens ? TokensSuffix : ParseTreeSuffix)}"), data);
}
public static SourceRepository CreateSourceRepository(string path, string tempDir,
CliParameters parameters)
{
SourceRepository sourceRepository;
if (string.IsNullOrWhiteSpace(path))
{
sourceRepository = DummySourceRepository.Instance;
}
else if (DirectoryExt.Exists(path))
{
sourceRepository = new DirectorySourceRepository(path);
}
else if (FileExt.Exists(path))
{
string extension = Path.GetExtension(path);
if (extension.EqualsIgnoreCase(".zip"))
{
var zipCachingRepository = new ZipCachingRepository(path);
if (tempDir != null)
{
zipCachingRepository.ExtractPath = tempDir;
}
sourceRepository = zipCachingRepository;
}
else
{
sourceRepository = new FileSourceRepository(path);
}
}
else
{
string url = path;
string projectName = null;
string urlWithoutHttp = TextUtils.HttpRegex.Replace(url, "");
if (!url.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
if (urlWithoutHttp.StartsWith("github.com", StringComparison.OrdinalIgnoreCase))
{
url = url + "/archive/master.zip";
}
}
if (urlWithoutHttp.StartsWith("github.com", StringComparison.OrdinalIgnoreCase))
{
projectName = urlWithoutHttp.Split('/').ElementAtOrDefault(2);
}
var zipAtUrlCachedCodeRepository = new ZipAtUrlCachingRepository(url, projectName);
if (tempDir != null)
{
zipAtUrlCachedCodeRepository.DownloadPath = tempDir;
}
sourceRepository = zipAtUrlCachedCodeRepository;
}
return(sourceRepository);
}
public void Save()
{
lock (saveLock)
{
DirectoryExt.CreateDirectory(Path.GetDirectoryName(settingsFileName));
FileExt.WriteAllText(settingsFileName, JsonConvert.SerializeObject(this, Formatting.Indented, jsonConverters));
}
}
/// <summary>
/// check each file in path to see if it's in the database, if not delete it
/// </summary>
/// <param name="path">path to check</param>
public uint DeleteOrphanedFiles()
{
uint deleted = 0;
path = path.TrimEnd('\\');
string[] files = DirectoryExt.GetFiles(path, "*.jpg;*.jpeg;*.png;*.bmp;*.gif");
for (int i = 0; i < files.Length; ++i)
{
byte[] data = File.ReadAllBytes(files[i]);
byte[] hash = ComputeHash(data);
string file = Path.GetFileName(files[i]);
if (file == "NA.JPG")
{
continue; // do delete NA.JPG files
}
// file exist but hash doesn't match
string sql = "SELECT id FROM art WHERE file=?file AND NOT hash=?hash";
MySqlCommand cmd = new MySqlCommand(sql);
cmd.Parameters.AddWithValue("?hash", hash);
cmd.Parameters.AddWithValue("?file", file);
object obj = db.ExecuteScalar(cmd);
if (obj != null)
{
// todo corrupt picture
}
// file doesn't exist
sql = "SELECT id FROM art WHERE file=?file";
cmd = new MySqlCommand(sql);
cmd.Parameters.AddWithValue("?hash", hash);
cmd.Parameters.AddWithValue("?file", file);
obj = db.ExecuteScalar(cmd);
if (obj == null)
{
File.Delete(files[i]);
string large = string.Format("{0}\\large\\{1}", path, file);
if (File.Exists(large))
{
File.Delete(large);
}
string small = string.Format("{0}\\small\\{1}", path, file);
if (File.Exists(small))
{
File.Delete(small);
}
string xsmall = string.Format("{0}\\xsmall\\{1}", path, file);
if (File.Exists(xsmall))
{
File.Delete(xsmall);
}
++deleted;
}
}
return(deleted);
}
private string PrepareRunDir(string baseDir)
{
string configDir = Path.Combine(_testResourceDir, baseDir);
string runDir = Path.Combine(_outDir, baseDir);
DirectoryExt.Delete(runDir);
DirectoryExt.Copy(configDir, runDir);
return(runDir);
}
/// <summary>
/// Extracts the contents of the sound package archive into the specified folder.
/// </summary>
/// <param name="folderPath"></param>
/// <param name="deleteExisting"></param>
public void ExtractToDirectory(string folderPath, bool deleteExisting = false)
{
if (deleteExisting && Directory.Exists(folderPath))
{
DirectoryExt.Delete(folderPath);
}
Archive.ExtractToDirectory(folderPath);
}
/// <summary>
/// Runs FictitiousPlay with the specified parameters.
/// Some parameters are set by default (e.g. verbosity), the caller has a chance to overwrite them
/// using Configure delegate.
/// </summary>
public StrategyTree[] Solve(ActionTree at, ChanceTree ct)
{
int playersCount = 2;
DirectoryExt.Delete(BaseDir);
Directory.CreateDirectory(BaseDir);
string inputDir = Path.Combine(BaseDir, "input");
Directory.CreateDirectory(inputDir);
string traceDir = Path.Combine(BaseDir, "trace");
string chanceTreeFile = Path.Combine(inputDir, "ct.dat");
string actionTreeFile = Path.Combine(inputDir, "at.dat");
ct.Write(chanceTreeFile);
at.Write(actionTreeFile);
if (VisualizeTrees)
{
VisActionTree.Show(at, actionTreeFile + ".gv");
VisChanceTree.Show(ct, chanceTreeFile + ".gv");
}
Solver.ChanceTreeFile = chanceTreeFile;
Solver.EqualCa = false;
Solver.ActionTreeFile = actionTreeFile;
Solver.OutputPath = BaseDir;
Solver.SnapshotsCount = 2;
Solver.Epsilon = Epsilon;
Solver.ThreadsCount = 6;
Solver.IsVerbose = true;
Solver.IterationVerbosity = 10000;
Solver.MaxIterationCount = 1000000000;
if (Configure != null)
{
Configure(Solver);
}
Solver.Solve();
StrategyTree[] eqStrategies = new StrategyTree[playersCount];
for (int p = 0; p < playersCount; ++p)
{
string fileName = Solver.CurrentSnapshotInfo.StrategyFile[p];
eqStrategies[p] = StrategyTree.Read <StrategyTree>(fileName);
if (VisualizeTrees)
{
VisStrategyTree.Show(eqStrategies[p], fileName + ".gv");
}
}
return(eqStrategies);
}
public void TestAssertEqualWhenEqual()
{
if (Directory.Exists(EqualDir))
{
Directory.Delete(EqualDir, true);
}
Assert.IsFalse(Directory.Exists(EqualDir));
DirectoryExt.CopyContents(ReferenceDir, EqualDir);
AssertDirectory.AreEqual(ReferenceDir, EqualDir, true);
}
void TestSort(int arraySize, int [] partSizes, int inMemSortSize, int rngSeed, int shuffleSize, bool isVerbose)
{
string inputDir = Path.Combine(_outDir, "input");
DirectoryExt.Delete(inputDir);
Directory.CreateDirectory(inputDir);
string tempDir = Path.Combine(_outDir, "temp");
int[] array = new int[arraySize];
for (int i = 0; i < array.Length; ++i)
{
array[i] = i;
}
SequenceRng rng = new SequenceRng(rngSeed);
rng.SetSequenceNoCopy(array);
rng.Shuffle(shuffleSize);
TestElement element = new TestElement();
int begin = 0;
for (int p = 0; p < partSizes.Length; ++p)
{
string fileName = Path.Combine(inputDir, string.Format("{0}.dat", p));
using (BinaryWriter writer = new BinaryWriter(new FileStream(fileName, FileMode.Create, FileAccess.Write)))
{
for (int i = begin; i < begin + partSizes[p]; ++i)
{
element.Write(array[i], writer);
}
begin += partSizes[p];
}
}
Assert.AreEqual(arraySize, begin, "Incorrect part sizes.");
string resultFile = Path.Combine(_outDir, "result.dat");
ExternalMergeSort <int> sorter = new ExternalMergeSort <int> {
IsVerbose = isVerbose
};
sorter.Sort(inputDir, tempDir, inMemSortSize, element, resultFile);
int[] result = ReadToArray(resultFile, element);
for (int i = 0; i < arraySize; ++i)
{
Assert.AreEqual(i, result[i]);
}
}
private void DownloadArchive()
{
bool fileDownloaded = false;
object progressLockObj = new object();
int previousPercent = 0;
long previousBytes = 0;
string currentFileName = Name;
WebClient webClient = new WebClient();
if (Logger != null)
{
webClient.DownloadProgressChanged += (sender, e) =>
{
lock (progressLockObj)
{
if (e.ProgressPercentage != 0)
{
if (e.ProgressPercentage / percentStep > previousPercent / percentStep)
{
Logger.LogInfo($"{currentFileName}: {e.ProgressPercentage / percentStep * percentStep}% downloaded.");
previousPercent = e.ProgressPercentage;
}
}
else
{
if (e.BytesReceived / bytesStep > previousBytes / bytesStep)
{
Logger.LogInfo($"{currentFileName}: {e.BytesReceived} bytes received.");
previousBytes = e.BytesReceived;
}
}
}
};
}
webClient.DownloadFileCompleted += (sender, e) => fileDownloaded = true;
Logger.LogInfo($"{Name} downloading...");
if (!DirectoryExt.Exists(DownloadPath))
{
DirectoryExt.CreateDirectory(DownloadPath);
}
webClient.DownloadFileAsync(new Uri(Url), ArchiveName);
do
{
Thread.Sleep(100);
}while (!fileDownloaded);
Logger.LogInfo($"{Name} downloaded.");
}
public void TestAssertEqualWhenDirectoryMissing()
{
if (Directory.Exists(MissingDirectoryDir))
{
Directory.Delete(MissingDirectoryDir, true);
}
Assert.IsFalse(Directory.Exists(MissingDirectoryDir));
DirectoryExt.CopyContents(ReferenceDir, MissingDirectoryDir);
var dirToDelete = Path.Combine(MissingDirectoryDir, "SubDir2");
Directory.Delete(dirToDelete, true);
Assert.Throws <AssertionException>(() => AssertDirectory.AreEqual(ReferenceDir, MissingDirectoryDir, true));
}
public void TestAssertEqualsWhenFileMissing()
{
if (Directory.Exists(MissingFileDir))
{
Directory.Delete(MissingFileDir, true);
}
Assert.IsFalse(Directory.Exists(MissingFileDir));
DirectoryExt.CopyContents(ReferenceDir, MissingFileDir);
var fileToDelete = Path.Combine(MissingFileDir, "SubDir1", "nunit.core.interfaces.dll");
File.Delete(fileToDelete);
Assert.Throws <AssertionException>(() => AssertDirectory.AreEqual(ReferenceDir, MissingFileDir, true));
}
protected bool IsDirectoryNotExistsOrEmpty()
{
if (!DirectoryExt.Exists(RootPath))
{
return(true);
}
IEnumerable <string> items = DirectoryExt.EnumerateFileSystemEntries(RootPath);
using (IEnumerator <string> en = items.GetEnumerator())
{
return(!en.MoveNext());
}
}
public IEnumerable <TextReader> OpenInputs()
{
if (InputFilename == null || !DirectoryExt.IncludesWildcard(InputFilename))
{
yield return(OpenInput());
}
else
{
foreach (var path in DirectoryExt.GetFiles(InputFilename))
{
yield return(OpenText(path));
}
}
}
public void TestAssertEqualWhenAdditionalFile()
{
if (Directory.Exists(AdditionalFileDir))
{
Directory.Delete(AdditionalFileDir, true);
}
Assert.IsFalse(Directory.Exists(AdditionalFileDir));
DirectoryExt.CopyContents(ReferenceDir, AdditionalFileDir);
var fileToCopy = Path.Combine(AdditionalFileDir, "AssemblyInfo.txt");
var newFilename = Path.Combine(AdditionalFileDir, "NewFile.txt");
File.Copy(fileToCopy, newFilename);
Assert.Throws <AssertionException>(() => AssertDirectory.AreEqual(ReferenceDir, AdditionalFileDir, true));
}
public override IEnumerable <string> GetFileNames()
{
if (RootPath == null)
{
return(Enumerable.Empty <string>());
}
IEnumerable <string> result = DirectoryExt.EnumerateFiles(RootPath, SearchPattern, SearchOption);
if (SearchPredicate != null)
{
result = result.Where(SearchPredicate);
}
return(result);
}
public void TestAssertEqualWhenMismatchedFile()
{
if (Directory.Exists(MismatchedFileDir))
{
Directory.Delete(MismatchedFileDir, true);
}
Assert.IsFalse(Directory.Exists(MismatchedFileDir));
DirectoryExt.CopyContents(ReferenceDir, MismatchedFileDir);
var filenameToModify = Path.Combine(MismatchedFileDir, "AssemblyInfo.txt");
var contents = File.ReadAllText(filenameToModify);
contents = contents + Environment.NewLine + "HELLO" + Environment.NewLine + contents;
File.Delete(filenameToModify);
File.WriteAllText(filenameToModify, contents);
Assert.Throws <AssertionException>(() => AssertDirectory.AreEqual(ReferenceDir, MismatchedFileDir, true));
}
/// <summary>
/// (Re)Scan and insert art from art directory.
/// This is a utility function that typically should not be used. This function
/// preforms the opposite of the function DeleteOrphanedFiles.
/// </summary>
private void RescanArt()
{
byte[] hash = null;
string[] files = DirectoryExt.GetFiles(Settings.Default.art_location, "*.jpg;*.jpeg;*.png;*.bmp;*.gif");
for (int i = 0; i < files.Length; ++i)
{
string filename = Path.GetFileName(files[i]);
string ext = Path.GetExtension(files[i]);
byte[] data = null;
string type = string.Empty;
string mime_type = string.Empty;
string description = string.Empty;
data = File.ReadAllBytes(files[i]);
type = "Cover";
mime_type = ext;
description = "cover art";
hash = ComputeHash(data);
string sql = "SELECT id FROM art WHERE hash=?hash";
MySqlCommand cmd = new MySqlCommand(sql);
cmd.Parameters.AddWithValue("?hash", hash);
object obj = null;
obj = db.ExecuteScalar(cmd);
try
{
obj = db.ExecuteScalar(cmd);
}
catch
{
continue;
}
if (obj == null)
{
sql = "INSERT INTO art VALUES(NULL, ?file, ?type, ?hash, ?description, ?mime_type, NULL, NOW())";
cmd = new MySqlCommand(sql);
cmd.Parameters.AddWithValue("?file", filename);
cmd.Parameters.AddWithValue("?type", type);
cmd.Parameters.AddWithValue("?hash", hash);
cmd.Parameters.AddWithValue("?description", description);
cmd.Parameters.AddWithValue("?mime_type", mime_type);
db.ExecuteNonQuery(cmd);
}
}
}
/// <summary>
/// Removes all temporary compiled files from the Compile directory
/// </summary>
public static void CleanCompileDirectory(IProgress <TaskProgressUpdate> progress)
{
if (Directory.Exists(CompilePath))
{
try
{
// Report
ProgressUpdate(progress, "Cleaning compile directory...");
// Delete Directory and create a new
DirectoryExt.Delete(CompilePath);
Directory.CreateDirectory(DefCompilePath);
}
catch (Exception e) when(e.Message.Contains("empty"))
{
throw new Exception("Unable to delete Compile directory because it is open. Please close this directory and try again.", e);
}
}
}
public SpriteMaster()
{
Self.AssertNull();
Self = this;
Garbage.EnterNonInteractive();
//SteamHelper.Init();
MemoryMonitor = new();
var assemblyPath = Assembly.Location;
assemblyPath = Path.GetDirectoryName(assemblyPath);
// Compress our own directory
if (assemblyPath is not null)
{
DirectoryExt.CompressDirectory(assemblyPath, force: true);
}
}
public static Settings Load()
{
if (FileExt.Exists(settingsFileName))
{
try
{
var settings = JsonConvert.DeserializeObject <Settings>(FileExt.ReadAllText(settingsFileName), jsonConverters) ?? new Settings();
return(settings);
}
catch
{
DirectoryExt.CreateDirectory(Path.GetDirectoryName(settingsFileName));
return(new Settings());
}
}
else
{
DirectoryExt.CreateDirectory(Path.GetDirectoryName(settingsFileName));
return(new Settings());
}
}
public static void OpenDirectory(string directoryName)
{
if (DirectoryExt.Exists(directoryName))
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Process.Start(new ProcessStartInfo {
FileName = "explorer.exe", Arguments = $"\"{directoryName}\""
});
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
Process.Start(new ProcessStartInfo {
FileName = "xdg-open", Arguments = directoryName, CreateNoWindow = true
});
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Process.Start(new ProcessStartInfo {
FileName = "open", Arguments = directoryName, CreateNoWindow = true
});
}
}
}
string CopyDirForEachBot(string runDir, int playerCount)
{
// Start from scratch
DeleteBotDirs(runDir);
string dirName = runDir + "-br";
DirectoryExt.Copy(runDir, dirName);
// For br: rename: strategy-{#}-br.dat -> strategy-{#}.dat
for (int p = 0; p < playerCount; ++p)
{
string fileName = Path.Combine(dirName, string.Format("strategy-{0}.dat", p));
// Check if this strategy was merged, if yes, skip it.
if (File.Exists(fileName))
{
File.Delete(fileName);
string fileName1 = Path.Combine(dirName, string.Format("strategy-{0}-br.dat", p));
File.Move(fileName1, fileName);
}
}
dirName = runDir + "-eq";
Directory.Move(runDir, dirName);
// For eq: remove br files
for (int p = 0; p < playerCount; ++p)
{
string fileName = Path.Combine(dirName, string.Format("strategy-{0}-br.dat", p));
// Check if this strategy was merged, if yes, skip it.
if (File.Exists(fileName))
{
File.Delete(fileName);
}
}
return(dirName);
}