private static void Main(string[] args)
{
var vifObjects = new VifObjects();
var fileEnumerator = new FileEnumerator(Path, "*.vif");
var builder = new VifObjectBuilder();
foreach (var path in fileEnumerator.EnumerateFiles())
{
//builder.Build(File.ReadAllLines(path));
}
}
/*
* Delete all temporary files from the codegen directory (e.g. source files, ...)
*/
internal void RemoveOldTempFiles()
{
Debug.Trace("BuildResultCache", "Deleting old temporary files from " + _cacheDir);
RemoveCodegenResourceDir();
string codegen = _cacheDir + "\\";
// Go through all the files in the codegen dir
foreach (FileData fileData in FileEnumerator.Create(codegen))
{
// Skip directories
if (fileData.IsDirectory)
{
continue;
}
// If it has a known extension, skip it
string ext = Path.GetExtension(fileData.Name);
if (ext == ".dll" || ext == ".pdb" || ext == ".web" || ext == ".ccu" || ext == ".prof" || ext == preservationFileExtension)
{
continue;
}
// .delete files need to be removed.
if (ext != dotDelete)
{
// Don't delete the temp file if it's named after a dll that's still around
// since it could still be useful for debugging.
// Note that we can't use GetFileNameWithoutExtension here because
// some of the files are named 5hvoxl6v.0.cs, and it would return
// 5hvoxl6v.0 instead of just 5hvoxl6v
int periodIndex = fileData.Name.LastIndexOf('.');
if (periodIndex > 0)
{
string baseName = fileData.Name.Substring(0, periodIndex);
int secondPeriodIndex = baseName.LastIndexOf('.');
if (secondPeriodIndex > 0)
{
baseName = baseName.Substring(0, secondPeriodIndex);
}
// Generated source files uses assemblyname as prefix so we should keep them.
if (FileUtil.FileExists(codegen + baseName + ".dll"))
{
continue;
}
// other generated files, such as .cmdline, .err and .out need to add the
// WebAssemblyNamePrefix, since they do not use the assembly name as prefix.
if (FileUtil.FileExists(codegen + BuildManager.WebAssemblyNamePrefix + baseName + ".dll"))
{
continue;
}
}
}
else
{
// Additional logic for VSWhidbey 564168 / Visual Studio QFE 4710.
// Delete both original .dll and .delete if possible
DiskBuildResultCache.CheckAndRemoveDotDeleteFile(new FileInfo(fileData.FullName));
continue;
}
Debug.Trace("BuildResultCache", "Deleting old temporary files: " + fileData.FullName);
try {
File.Delete(fileData.FullName);
} catch { }
}
}
public BaseCommand()
{
_fileReader = new FileReader();
_fileWriter = new FileWriter();
_fileEnumerator = new FileEnumerator();
}
/// <summary>
/// Runs this command.
/// </summary>
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("PARTITION\n");
Console.ResetColor();
Console.WriteLine(
$"Input dir: {_inputDir}\n" +
$"Input mask: {_fileMask}\n" +
$"Output dir: {_outputDir}\n" +
$"Min: {_minTreshold}\n" +
$"Max: {_maxTreshold}\n" +
$"Recursive: {_recursive}\n");
Log.Logger.Information("PARTITION");
XmlPartitioner partitioner = new XmlPartitioner
{
MinTreshold = _minTreshold,
MaxTreshold = _maxTreshold
};
int partitioned = 0, total = 0;
if (!Directory.Exists(_outputDir))
{
Directory.CreateDirectory(_outputDir);
}
foreach (string filePath in FileEnumerator.Enumerate(
_inputDir, _fileMask, _regexMask, _recursive))
{
total++;
Console.Write(filePath);
XDocument doc = XDocument.Load(filePath,
LoadOptions.PreserveWhitespace);
bool touched = partitioner.Partition(doc,
Path.GetFileNameWithoutExtension(filePath));
string outputPath =
Path.Combine(_outputDir, Path.GetFileName(filePath));
if (touched)
{
partitioned++;
Console.WriteLine($" => {outputPath}");
if (!Directory.Exists(_outputDir))
{
Directory.CreateDirectory(_outputDir);
}
doc.Save(outputPath, SaveOptions.OmitDuplicateNamespaces);
}
else
{
File.Copy(filePath, outputPath);
Console.WriteLine();
}
}
Console.WriteLine($"Total files: {total}");
Console.WriteLine($"Partitioned files: {partitioned}");
return(Task.CompletedTask);
}
private void BeginScanning()
{
if (ProcessingToggle.CurrentState == ToggleState.Active)
{
btnSearch.Enabled = false;
ProcessingToggle.SetState(ToggleState.Inactive);
}
else if (ProcessingToggle.CurrentState == ToggleState.Ready)
{
btnSearch.Enabled = false;
ProcessingToggle.SetState(ToggleState.Active);
bool calculateEntropy = checkboxCalculateEntropy.Checked;
string selectedFolder = tbPath.Text;
string searchPatterns = tbSearchPatterns.Text;
List <YaraFilter> yaraParameters = new List <YaraFilter>();
if (checkBoxYaraRules.Checked)
{
yaraParameters = currentYaraFilters.ToList();
}
IDataPersistenceLayer dataPersistenceLayer = null;
if (radioPersistenceCSV.Checked)
{
dataPersistenceLayer = new CsvDataPersistenceLayer(tbPersistenceParameter.Text);
}
else if (radioPersistenceSqlite.Checked)
{
dataPersistenceLayer = new SqliteDataPersistenceLayer(tbPersistenceParameter.Text);
}
else if (radioPersistenceSqlServer.Checked)
{
dataPersistenceLayer = new SqlDataPersistenceLayer(tbPersistenceParameter.Text);
}
FileEnumeratorParameters parameters =
new FileEnumeratorParameters(cancelToken,
Settings.FileEnumeration_DisableWorkerThread,
selectedFolder,
searchPatterns,
calculateEntropy,
yaraParameters,
dataPersistenceLayer,
Log.ToUI, Log.ToFile, ReportNumbers, Log.ExceptionMessage);
enumerationStart = DateTime.Now;
bool didThrow = false;
try
{
parameters.ThrowIfAnyParametersInvalid();
}
catch (Exception ex)
{
didThrow = true;
MessageBox.Show(
ex.ToString().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(),
MsgBox_TitleBarText,
MessageBoxButtons.OK,
MessageBoxIcon.Error
);
}
if (didThrow)
{
ProcessingToggle.SetState(ToggleState.Ready);
}
else
{
Log.ToUI(Environment.NewLine);
Log.ToAll($"Beginning Enumeration of folder: \"{parameters.SelectedFolder}\"");
Log.ToAll("Parsing MFT. (This may take a few minutes)");
FileEnumerator.LaunchFileEnumerator(parameters);
}
}
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("IMPORT JSON TEXT AND APPARATUS\n");
Console.ResetColor();
Console.WriteLine(
$"Text dir: {_txtFileDir}\n" +
$"Text mask: {_txtFileMask}\n" +
$"Apparatus dir: {_appFileDir}\n" +
$"Profile file: {_profilePath}\n" +
$"Database: {_database}\n" +
$"Dry run: {_dry}\n");
ILoggerFactory loggerFactory = new LoggerFactory();
loggerFactory.AddSerilog(Log.Logger);
Log.Logger.Information("IMPORT JSON TEXT AND APPARATUS");
if (!_dry)
{
// create database if not exists
string connection = string.Format(CultureInfo.InvariantCulture,
_config.GetConnectionString("Mongo"),
_database);
IDatabaseManager manager = new MongoDatabaseManager();
string profileContent = LoadProfile(_profilePath);
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
DataProfile profile = serializer.Read(profileContent);
if (!manager.DatabaseExists(connection))
{
Console.WriteLine("Creating database...");
Log.Information($"Creating database {_database}...");
manager.CreateDatabase(connection, profile);
Console.WriteLine("Database created.");
Log.Information("Database created.");
}
}
else
{
if (!File.Exists(_profilePath))
{
string error = "Profile path not found: " + _profilePath;
Console.WriteLine(error);
Log.Error(error);
return(Task.CompletedTask);
}
}
ICadmusRepository repository =
_repositoryService.CreateRepository(_database);
JsonImporter importer = new JsonImporter(repository)
{
Logger = loggerFactory.CreateLogger("json-importer"),
IsDry = _dry
};
int inputFileCount = 0;
// 1) import text
string[] files = FileEnumerator.Enumerate(
_txtFileDir, _txtFileMask, _regexMask).ToArray();
HashSet <string> fileNames = new HashSet <string>();
Console.WriteLine($"Importing text from {files.Length} file(s)...");
foreach (string txtFilePath in files)
{
fileNames.Add(
StripFileNameNr(
Path.GetFileNameWithoutExtension(txtFilePath)));
Console.WriteLine(txtFilePath);
inputFileCount++;
using (Stream stream = new FileStream(txtFilePath, FileMode.Open,
FileAccess.Read, FileShare.Read))
{
importer.ImportText(stream);
}
}
// 2) import apparatus
Console.WriteLine("Importing apparatus...");
foreach (string fileName in fileNames)
{
Console.WriteLine(fileName);
foreach (string appFilePath in Directory.EnumerateFiles(
_appFileDir, fileName + "-app_*.json"))
{
Console.WriteLine(" " + appFilePath);
using (Stream stream = new FileStream(appFilePath, FileMode.Open,
FileAccess.Read, FileShare.Read))
{
importer.ImportApparatus(stream);
}
}
}
return(Task.CompletedTask);
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("PARSE APPARATUS\n");
Console.ResetColor();
Console.WriteLine(
$"Input: {_inputFileMask}\n" +
$"Output: {_outputDir}\n" +
$"Max items per file: {_maxItemPerFile}\n");
ILoggerFactory loggerFactory = new LoggerFactory();
loggerFactory.AddSerilog(Log.Logger);
Log.Logger.Information("PARSE APPARATUS");
XmlApparatusParser parser = new XmlApparatusParser
{
Logger = loggerFactory.CreateLogger("parse-app")
};
int inputFileCount = 0;
int totalPartCount = 0;
StreamWriter writer = null;
if (!Directory.Exists(_outputDir))
{
Directory.CreateDirectory(_outputDir);
}
// for each input document
foreach (string filePath in FileEnumerator.Enumerate(
_inputFileDir, _inputFileMask, _regexMask, _recursive))
{
Console.WriteLine();
Log.Logger.Information("Parsing {FilePath}", filePath);
// load document
string inputFileName = Path.GetFileNameWithoutExtension(filePath);
Console.WriteLine(filePath);
inputFileCount++;
XDocument doc = XDocument.Load(filePath,
LoadOptions.PreserveWhitespace | LoadOptions.SetLineInfo);
JsonSerializerSettings jsonSettings = new JsonSerializerSettings
{
ContractResolver = new DefaultContractResolver
{
NamingStrategy = new CamelCaseNamingStrategy()
},
Formatting = Formatting.Indented
};
// load index
string textFileName = inputFileName.Replace("-app", "");
LoadTextIndex(textFileName);
// parse
int partCount = 0, outputFileCount = 0;
foreach (var part in parser.Parse(doc, textFileName, _textIndex))
{
if (++partCount % 10 == 0)
{
Console.Write('.');
}
// create new output file if required
if (writer == null ||
(_maxItemPerFile > 0 && partCount > _maxItemPerFile))
{
if (writer != null)
{
CloseOutputFile(writer);
}
string path = Path.Combine(_outputDir,
$"{inputFileName}_{++outputFileCount:00000}.json");
writer = new StreamWriter(new FileStream(path,
FileMode.Create, FileAccess.Write, FileShare.Read),
Encoding.UTF8);
writer.WriteLine("[");
}
// dump part into it
string json = JsonConvert.SerializeObject(part, jsonSettings);
writer.WriteLine(json + ",");
}
totalPartCount += partCount;
if (writer != null)
{
CloseOutputFile(writer);
writer = null;
}
}
Console.WriteLine($"\nInput documents: {inputFileCount}");
Console.WriteLine($"Output parts: {totalPartCount}");
return(Task.CompletedTask);
}
/// <summary>
/// Goes through the Directories to scan recursively and starts a scan of each while that matches the whitelist. (both from settings)
/// </summary>
/// <returns>All found nuggets.</returns>
public IDictionary <string, TemplateItem> ParseAll()
{
IEnumerable <string> fileWhiteList = _settings.WhiteList;
IEnumerable <string> directoriesToSearchRecursively = _settings.DirectoriesToScan;
FileEnumerator fileEnumerator = new FileEnumerator(_settings.BlackList);
string currentFullPath;
bool blacklistFound = false;
var templateItems = new ConcurrentDictionary <string, TemplateItem>();
// Collection of template items keyed by their id.
foreach (var directoryPath in directoriesToSearchRecursively)
{
foreach (string filePath in fileEnumerator.GetFiles(directoryPath))
{
if (filePath.Length >= 260)
{
DebugHelpers.WriteLine("Path too long to process. Path: " + filePath);
continue;
}
blacklistFound = false;
currentFullPath = Path.GetDirectoryName(Path.GetFullPath(filePath));
foreach (var blackItem in _settings.BlackList)
{
if (currentFullPath == null || currentFullPath.StartsWith(blackItem, StringComparison.OrdinalIgnoreCase))
{
//this is a file that is under a blacklisted directory so we do not parse it.
blacklistFound = true;
break;
}
}
if (!blacklistFound)
{
//we check every filePath against our white list. if it's on there in at least one form we check it.
foreach (var whiteListItem in fileWhiteList)
{
//We have a catch all for a filetype
if (whiteListItem.StartsWith("*."))
{
if (Path.GetExtension(filePath) == whiteListItem.Substring(1))
{
//we got a match
ParseFile(_settings.ProjectDirectory, filePath, templateItems);
break;
}
}
else //a file, like myfile.js
{
if (Path.GetFileName(filePath) == whiteListItem)
{
//we got a match
ParseFile(_settings.ProjectDirectory, filePath, templateItems);
break;
}
}
}
}
}
}
return(templateItems);
}
internal MapPathBasedVirtualPathEnumerator(VirtualPath virtualPath, RequestedEntryType requestedEntryType)
{
if (virtualPath.IsRelative)
{
throw new ArgumentException(SR.GetString(SR.Invalid_app_VirtualPath), "virtualPath");
}
_virtualPath = virtualPath;
_requestedEntryType = requestedEntryType;
string physicalPath;
if (!ServerConfig.UseServerConfig)
{
// Use the hosting environment to map the virtual path
physicalPath = _virtualPath.MapPathInternal();
}
else
{
IServerConfig serverConfig = ServerConfig.GetInstance();
_serverConfig2 = serverConfig as IServerConfig2;
// Use serverConfig to map the virtual path
physicalPath = serverConfig.MapPath(null, _virtualPath);
if (_requestedEntryType != RequestedEntryType.Files)
{
// For MetabaseServerConfig, get the subdirs that are not in the application, and add them to the exclude list.
if (_serverConfig2 == null)
{
string [] virtualSubdirsNotInApp = serverConfig.GetVirtualSubdirs(_virtualPath, false);
if (virtualSubdirsNotInApp != null)
{
_exclude = new Hashtable(StringComparer.OrdinalIgnoreCase);
foreach (string subdir in virtualSubdirsNotInApp)
{
_exclude[subdir] = subdir;
}
}
}
// Get subdirs that are virtual directories, and record their physical mappings.
// Ignore the virtualPaths if we only need files, since it only contains directories
string [] virtualSubdirsInApp = serverConfig.GetVirtualSubdirs(_virtualPath, true);
if (virtualSubdirsInApp != null)
{
_virtualPaths = new Hashtable(StringComparer.OrdinalIgnoreCase);
foreach (string subdir in virtualSubdirsInApp)
{
VirtualPath subpath = _virtualPath.SimpleCombineWithDir(subdir);
string subPhysicalPath = serverConfig.MapPath(null, subpath);
if (FileUtil.DirectoryExists(subPhysicalPath))
{
_virtualPaths[subdir] = new MapPathBasedVirtualDirectory(subpath.VirtualPathString);
}
}
// Create enumerator for the virtual paths
_virtualEnumerator = _virtualPaths.Values.GetEnumerator();
}
}
}
// Create an enumerator for the physical files and directories at this path
_fileEnumerator = FileEnumerator.Create(physicalPath);
// Reset the enumerator. Note that we don't support the Reset method.
_useFileEnumerator = false;
}
public IEnumerator GetEnumerator()
{
return(fileEnumerator = new FileEnumerator(this.filepath));
}
/// <summary>
/// Creates a <see cref="BackgroundWorker"/> class instance to update the song locations with a user selected path.
/// </summary>
/// <param name="folderBrowserDialog">An instance to <see cref="Ookii.Dialogs.WinForms.VistaFolderBrowserDialog"/> for the user to select the path.</param>
/// <param name="connection">A SQLite connection to use for the procedure.</param>
/// <returns>An instance to a <see cref="BackgroundWorker"/> class to handle the operation.</returns>
public static BackgroundWorker UpdateSongLocations(
VistaFolderBrowserDialog folderBrowserDialog, SQLiteConnection connection)
{
if (folderBrowserDialog.ShowDialog() == DialogResult.OK)
{
BackgroundWorker worker = new BackgroundWorker
{
WorkerReportsProgress = true, WorkerSupportsCancellation = true
};
worker.DoWork += (sender, args) =>
{
List <FileEnumeratorFileEntry> files = new List <FileEnumeratorFileEntry>(FileEnumerator
.RecurseFiles(folderBrowserDialog.SelectedPath,
Constants.Extensions.ToArray()).ToArray());
if (files.Count == 0)
{
worker.ReportProgress(100);
return;
}
int fileCount = files.Count;
long songCount = GetScalar <long>(Song.GenerateCountSentence(false), connection);
int progress = 0;
int previousProgress = -1;
List <Song> songs = new List <Song>();
using (SQLiteCommand command = new SQLiteCommand(Song.GenerateSelectSentence(false), connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
var percentage = progress * 50 / (int)songCount;
if (percentage > previousProgress)
{
worker.ReportProgress(percentage);
previousProgress = percentage;
}
songs.Add(Song.FromSqLiteDataReader(reader));
progress++;
if (worker.CancellationPending)
{
return;
}
}
}
}
progress = 0;
foreach (var file in files)
{
try
{
var fileInfo = new FileInfo(file.FileNameWithPath);
var song = songs.FirstOrDefault(f =>
f.FileSize == fileInfo.Length && f.FileNameNoPath == fileInfo.Name);
if (song == null)
{
song = songs.FirstOrDefault(f => f.Filename == fileInfo.FullName);
if (song != null)
{
song.FileSize = (int)fileInfo.Length;
}
else
{
song = songs.FirstOrDefault(f => f.FileNameNoPath == fileInfo.Name);
if (song != null)
{
song.FileSize = (int)fileInfo.Length;
}
}
}
var percentage = 50 + progress * 50 / fileCount;
if (percentage > previousProgress)
{
worker.ReportProgress(percentage);
previousProgress = percentage;
}
if (song == null)
{
continue;
}
song.Filename = file.FileNameWithPath;
ExecuteSql(song.GenerateInsertUpdateSqlSentence(false), connection);
progress++;
if (worker.CancellationPending)
{
return;
}
}
catch (Exception ex)
{
// log the exception..
ExceptionLogger.LogError(ex);
}
}
worker.ReportProgress(100);
};
return(worker);
}
return(null);
}
private static void Main(string[] args)
{
string connectionString = Settings.Database_ConnectionString;
if (string.IsNullOrWhiteSpace(connectionString) || connectionString == "SetMe")
{
ReportOutput("ERROR: Connection string not set! Please set the SQL connection string in .config file.");
ReportOutput("Aborting...");
return;
}
else
{
FilePropertiesAccessLayer.SetConnectionString(connectionString);
}
if (args.Length == 0)
{
DisplayUsageSyntax();
return;
}
// Will hold flag & parameter to flag, such as: "-p", "C:\Windows\"
List <Tuple <string, string> > flags = GetFlags(args);
if (!flags.Any())
{
return;
}
string searchPath = "";
string searchMask = "*.*";
bool calcEntropy = false;
bool onlineValidation = false;
string yaraRulesFile = "";
foreach (Tuple <string, string> flagTuple in flags)
{
string flag = flagTuple.Item1;
string parameter = flagTuple.Item2;
switch (flag)
{
case "e":
calcEntropy = true;
break;
case "v":
onlineValidation = true;
break;
case "p":
searchPath = parameter;
break;
case "m":
searchMask = parameter;
break;
case "y":
yaraRulesFile = parameter;
break;
}
}
ReportOutput($"Search [P]ath: \"{searchPath}\"");
ReportOutput($"Search [M]ask: {searchMask}");
ReportOutput($"Calulate [E]ntropy: {calcEntropy}");
ReportOutput($"Online [V]alidation: {onlineValidation}");
ReportOutput($"[Y]ara Rules File: \"{yaraRulesFile}\"");
ReportOutput("");
FileEnumeratorParameters parameters = new FileEnumeratorParameters(CancellationToken.None, Settings.FileEnumeration_DisableWorkerThread, searchPath, searchMask, calcEntropy, onlineValidation, yaraRulesFile, ReportOutput, LogOutput, ReportResults, ReportException);
ReportOutput("Beginning enumeration...");
FileEnumerator.LaunchFileEnumerator(parameters);
}
public void EnumerateFilesWithoutExtension_returns_empty_when_pass_in_null()
{
FileEnumerator.EnumerateFilesWithoutExtension(null).Should().BeEmpty();
}
/// <summary>
/// Searches the TMDb database for a TV season based on a given path using a TMdbEasy.EasyClient class instance.
/// </summary>
/// <param name="easy">A TMdbEasy.EasyClient class instance to use for the search.</param>
/// <param name="path">A path to enumerate files from.</param>
/// <param name="stillSize">The size of the still image to get the URL for.</param>
/// <param name="excludeFileNames">File names to be left out of the search.</param>
/// <returns>A collection of TMDbDetail class instances containing the TV show season information from the TMDb database.</returns>
public static async Task <IEnumerable <TMDbDetail> > GetSeasonAsync(EasyClient easy, string path, string stillSize = "original", List <string> excludeFileNames = null)
{
var televisionApi = easy.GetApi <ITelevisionApi>().Value; // create a ITelevisionApi..
// avoid the null value..
excludeFileNames = excludeFileNames ?? new List <string>();
string fileName = string.Empty;
if (File.Exists(path))
{
fileName = path;
path = Path.GetDirectoryName(path);
}
// List files of known video formats from the given path..
IEnumerable <FileEnumeratorFileEntry> fileEntries =
await FileEnumerator.RecurseFilesAsync(path, FileEnumerator.FiltersVideoVlcNoBinNoIso).ConfigureAwait(false);
// remove the "useless" files from the list..
List <FileEnumeratorFileEntry> tmpFilesList = new List <FileEnumeratorFileEntry>();
// not in the include list..
if (fileName != string.Empty)
{
fileEntries = fileEntries.Where(f => f.FileNameWithPath == fileName);
}
foreach (FileEnumeratorFileEntry entry in fileEntries.ToList())
{
// excluded from the list..
if (excludeFileNames.Contains(entry.FileName))
{
continue; // ..so do continue..
}
tmpFilesList.Add(entry); // ..else add to the list..
}
fileEntries = tmpFilesList; // re-assign the list back to the source..
// don't start searching if the directory is empty - we don't want to cause excess stress for the TMDb database..
if (fileEntries.ToList().Count == 0)
{
// ..so just throw an exception..
throw new Exception("No files were found from the given path string.");
}
// construct a search string of the given path..
string searchString = GetTVShowSearchString(path);
int season = GetTVShowSeason(path); // extract a season number from the given path..
if (season == -1) // if no season number was in the given path..
{
// ..just throw an exception..
throw new Exception("The TV season number wasn't found of the given path string.");
}
// initialize the result..
List <TMDbDetail> result = new List <TMDbDetail>();
// search for TV shows base on the search string build from the given directory name..
TVShowList list = await televisionApi.SearchTVShowsAsync(searchString).ConfigureAwait(false);
// if something was found..
if (list != null && list.Total_results > 0) // ..deepen the search..
{
string seriesName = list.Results[0].Name; // save the name of the TV show..
int seriesID = list.Results[0].Id; // save the ID of the TV show..
// find the TV show season from the TMDb database with an ID and a season number..
TvSeason tvSeason = await televisionApi.GetSeasonDetailsAsync(seriesID, season).ConfigureAwait(false);
// if something was found..
if (tvSeason != null && tvSeason.Episodes != null)
{
foreach (Episode episode in tvSeason.Episodes) // return the details of the TV show season..
{
// don't return a file-less TMDbDetail class instance..
if (GetFileNameMatchingTVSeasonEpisode(fileEntries, season, episode.Episode_number) == null)
{
continue; // ..so just continue the loop..
}
result.Add(new TMDbDetail
{
ID = seriesID, // the TMDb id for the TV show..
SeasonID = tvSeason.Id, // the TMDb id for the TV show season..
EpisodeID = episode.Id, // the TMDb id for the TV show season episode..
// formulate the title base on the TVEpisodeFormat or the "overriding" TVEpisodeFormatCustom property value..
Title = TVEpisodeFormatCustom != string.Empty ?
TVEpisodeFormatCustom.
Replace("#SNUM#", tvSeason.Season_number.ToString()). // the season number as one digit..
Replace("#ENUM#", episode.Episode_number.ToString()). // the episode number as one digit..
Replace("#EPISODE_NAME#", episode.Name). // the name of the episode..
Replace("#SERIES_NAME#", seriesName). // the name of the series..
Replace("#SEASON_NAME#", tvSeason.Name). // the name of the TV show season..
Replace("#SNUM2#", string.Format("{0:00}", tvSeason.Season_number)). // the season number as two digits..
Replace("#ENUM2#", string.Format("{0:00}", episode.Episode_number)) : // the episode number as two digits..
string.Format(TVEpisodeFormat, seriesName, tvSeason.Name, episode.Episode_number, episode.Name),
// set the description..
Description = string.IsNullOrEmpty(tvSeason.Overview) ? episode.Overview : tvSeason.Overview,
DetailDescription = episode.Overview, // set the detailed description if any..
// find the file name for the TV show episode..
FileName = GetFileNameMatchingTVSeasonEpisode(fileEntries, season, episode.Episode_number).FileName,
// create an URL for the still using the TV season's poster path as a "fail safe"..
PosterOrStillURL = new Uri("https://image.tmdb.org/t/p/" + stillSize +
(string.IsNullOrEmpty(episode.Still_path) ? tvSeason.Poster_path : episode.Still_path)),
Season = season, // set the season number..
Episode = episode.Episode_number // set the episode number..
});
}
}
else // nothing was found..
{
// loop through the found files..
foreach (FileEnumeratorFileEntry entry in fileEntries)
{
result.Add(new TMDbDetail
{
Title = entry.FileNameNoExtension, // the title can be the file name without path or extension..
FileName = entry.FileName, // set the file name..
Season = season, // set the season number..
Episode = GetTVShowEpisodeNumber(entry.FileName) // set the episode number..
});
}
}
}
return(result);
}
/// <summary>
/// Upload a directory to Azure File Storage.
/// </summary>
/// <param name="sourcePath">Path to the source directory</param>
/// <param name="destFileDir">The <see cref="CloudFileDirectory"/> that is the destination Azure file directory.</param>
/// <param name="options">An <see cref="UploadDirectoryOptions"/> object that specifies additional options for the operation.</param>
/// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
/// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
public static Task UploadDirectoryAsync(string sourcePath, CloudFileDirectory destFileDir, UploadDirectoryOptions options, TransferContext context, CancellationToken cancellationToken)
{
DirectoryLocation sourceLocation = new DirectoryLocation(sourcePath);
AzureFileDirectoryLocation destLocation = new AzureFileDirectoryLocation(destFileDir);
FileEnumerator sourceEnumerator = new FileEnumerator(sourceLocation);
if (options != null)
{
sourceEnumerator.SearchPattern = options.SearchPattern;
sourceEnumerator.Recursive = options.Recursive;
}
return UploadDirectoryInternalAsync(sourceLocation, destLocation, sourceEnumerator, options, context, cancellationToken);
}
private static void Main(string[] args)
{
if (args.Length == 0)
{
DisplayUsageSyntax();
return;
}
// Will hold flag & parameter to flag, such as: "-p", "C:\Windows\"
List <Tuple <string, string> > flags = GetFlags(args);
if (!flags.Any())
{
DisplayUsageSyntax();
return;
}
string searchPath = "";
string searchMask = "*.*";
bool isEntropyEnabled = false;
bool isYaraEnabled = false;
bool isSqlServerEnabled = false;
bool isSqliteEnabled = false;
bool isCsvEnabled = false;
string sqliteDbFile = "";
string csvFile = "";
string sqlConnectionString = (Settings.Database_ConnectionString == "SetMe") ? "" : Settings.Database_ConnectionString;
string yaraFiltersFile = "";
List <YaraFilter> yaraFilters = new List <YaraFilter>();
foreach (Tuple <string, string> flagTuple in flags)
{
string flag = flagTuple.Item1;
string parameter = flagTuple.Item2;
switch (flag)
{
case "e":
isEntropyEnabled = true;
break;
case "p":
searchPath = parameter;
break;
case "m":
searchMask = parameter;
break;
case "y":
isYaraEnabled = true;
yaraFiltersFile = parameter;
break;
case "s":
isSqlServerEnabled = true;
break;
case "l":
isSqliteEnabled = true;
sqliteDbFile = string.IsNullOrWhiteSpace(parameter) ? sqlConnectionString : parameter;
break;
case "c":
isCsvEnabled = true;
csvFile = parameter;
break;
}
}
ReportOutput();
ReportOutput("Running with these parameters:");
ReportOutput($" Search [P]ath: \"{searchPath}\"");
ReportOutput($" Search [M]ask: {searchMask}");
ReportOutput($" Calulate [E]ntropy: {isEntropyEnabled}");
if (isYaraEnabled)
{
ReportOutput($" [Y]ara filters file: \"{yaraFiltersFile}\"");
}
if (isSqlServerEnabled)
{
ReportOutput($" [S]QL connection: \"{sqlConnectionString}\"");
}
else if (isSqliteEnabled)
{
ReportOutput($" Sq[l]ite DB: \"{sqliteDbFile}\"");
}
else if (isCsvEnabled)
{
ReportOutput($" [C]SV file: \"{csvFile}\"");
}
ReportOutput();
if (string.IsNullOrWhiteSpace(searchPath))
{
ReportOutput("No search path provided!");
ReportOutput("You must supply the -p flag with a path, e.g.:");
ReportOutput("-p:\"C:\\Program Files\\BanzaiBuddy\"");
ReportOutput();
ReportOutput("Aborting...");
return;
}
if (isYaraEnabled)
{
if (!File.Exists(yaraFiltersFile))
{
ReportOutput($"The yara filters file path suppled does not exist: \"{yaraFiltersFile}\".");
ReportOutput();
ReportOutput("Aborting...");
return;
}
try
{
string loadJson = File.ReadAllText(yaraFiltersFile);
yaraFilters = JsonConvert.DeserializeObject <List <YaraFilter> >(loadJson);
}
catch
{
ReportOutput("The yara filters file must be a JSON file.");
ReportOutput();
ReportOutput("Aborting...");
return;
}
}
IDataPersistenceLayer dataPersistenceLayer;
if (isSqliteEnabled)
{
dataPersistenceLayer = new SqliteDataPersistenceLayer(sqliteDbFile);
}
else if (isCsvEnabled)
{
dataPersistenceLayer = new CsvDataPersistenceLayer(csvFile);
}
else if (isSqlServerEnabled)
{
dataPersistenceLayer = new SqlDataPersistenceLayer(sqlConnectionString);
}
else
{
ReportOutput("No output parameter provided!");
if (!string.IsNullOrWhiteSpace(sqlConnectionString))
{
ReportOutput("(SQL server connection string supplied in config file, asuming SQL server output...)");
dataPersistenceLayer = new SqlDataPersistenceLayer(sqlConnectionString);
}
else
{
ReportOutput("You must supply an output parameter, e.g.:");
ReportOutput("-c:C:\\out.csv");
ReportOutput($"OR provide a SQL server connection string in the config file: {_thisExecutableFilename}");
ReportOutput("(Because it defaults to a SQL server connection. However, the connection string was missing.)");
ReportOutput();
ReportOutput("Aborting...");
return;
}
}
FileEnumeratorParameters parameters =
new FileEnumeratorParameters(
CancellationToken.None,
true, // Do not change this. If set to false, it will run on a thread, return immediately and exit, killing the thread.
searchPath,
searchMask,
isEntropyEnabled,
yaraFilters,
dataPersistenceLayer,
ReportOutput,
Log.LogOutputAction,
ReportResults,
Log.ExceptionMessage
);
parameters.ThrowIfAnyParametersInvalid();
ReportOutput("Beginning scan...");
FileEnumerator.LaunchFileEnumerator(parameters);
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("REPORT OVERLAPS\n");
Console.ResetColor();
Console.WriteLine(
$"Input: {_appFileMask}\n" +
$"Output: {_outputPath}\n");
int inputFileCount = 0;
int overlapCount = 0;
ILoggerFactory loggerFactory = new LoggerFactory();
loggerFactory.AddSerilog(Log.Logger);
Log.Logger.Information("REPORT OVERLAPS");
using (StreamWriter writer = new StreamWriter(_outputPath, false,
Encoding.UTF8))
{
writer.WriteLine("# Overlaps Report");
writer.WriteLine();
writer.WriteLine($"Input: `{_appFileDir}{Path.DirectorySeparatorChar}{_appFileMask}`");
writer.WriteLine();
// for each app document
WordIdList widList = new WordIdList
{
Logger = loggerFactory.CreateLogger("report-overlaps")
};
foreach (string filePath in FileEnumerator.Enumerate(
_appFileDir, _appFileMask, _regexMask, _recursive))
{
Console.WriteLine();
Log.Logger.Information("Parsing {FilePath}", filePath);
// load app document
string inputFileName = Path.GetFileNameWithoutExtension(filePath);
Console.WriteLine(filePath);
inputFileCount++;
XDocument doc = XDocument.Load(filePath,
LoadOptions.PreserveWhitespace | LoadOptions.SetLineInfo);
// collect word IDs from text document
widList.Parse(XDocument.Load(filePath.Replace("-app.", ".")));
// collect app's locations
List <AppElemLocations> appElemLocs =
AppElemLocationCollector.Collect(doc, widList,
AppElemLocationCollector.IsOverlappable);
// detect and report overlaps
for (int i = 0; i < appElemLocs.Count - 1; i++)
{
for (int j = i + 1; j < appElemLocs.Count; j++)
{
if (appElemLocs[i].Overlaps(appElemLocs[j]))
{
writer.WriteLine($"## Overlap {++overlapCount}");
writer.WriteLine();
writer.WriteLine(Path.GetFileName(filePath) +
$" at {appElemLocs[i].LineNumber}");
// text
int n = 0;
foreach (var iw in appElemLocs[i].Locations)
{
if (++n > 1)
{
writer.Write(' ');
}
writer.Write($"`{iw.Item1}`=`{iw.Item2}`");
}
writer.WriteLine();
writer.WriteLine();
// app
WriteAppXml(appElemLocs[i], writer);
WriteAppXml(appElemLocs[j], writer);
goto nextOuter;
}
}
nextOuter:
if (i % 10 == 0)
{
Console.Write('.');
}
}
Console.WriteLine();
}
writer.Flush();
}
Console.WriteLine($"\nInput documents: {inputFileCount}");
return(Task.CompletedTask);
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("REMOVE OVERLAPS\n");
Console.ResetColor();
Console.WriteLine(
$"Input: {_appFileMask}\n" +
$"Output: {_outputDir}\n" +
$"Div list: {(_writeDivList ? "yes" : "no")}\n");
int inputFileCount = 0;
int removedCount = 0;
ILoggerFactory loggerFactory = new LoggerFactory();
loggerFactory.AddSerilog(Log.Logger);
Log.Logger.Information("REMOVE OVERLAPS");
if (!Directory.Exists(_outputDir))
{
Directory.CreateDirectory(_outputDir);
}
HashSet <Tuple <string, string> > errDivIds =
new HashSet <Tuple <string, string> >();
// for each app document
WordIdList widList = new WordIdList
{
Logger = loggerFactory.CreateLogger("report-overlaps")
};
foreach (string filePath in FileEnumerator.Enumerate(
_appFileDir, _appFileMask, _regexMask, _recursive))
{
Console.WriteLine();
Log.Logger.Information("Parsing {FilePath}", filePath);
string docId = Path.GetFileNameWithoutExtension(filePath)
.Replace("-app", "");
// load app document
string inputFileName = Path.GetFileNameWithoutExtension(filePath);
Console.WriteLine(filePath);
inputFileCount++;
XDocument doc = XDocument.Load(filePath,
LoadOptions.PreserveWhitespace | LoadOptions.SetLineInfo);
// collect word IDs from text document
widList.Parse(XDocument.Load(filePath.Replace("-app.", ".")));
// collect app's locations
List <AppElemLocations> appElemLocs =
AppElemLocationCollector.Collect(doc, widList,
AppElemLocationCollector.IsOverlappable);
// detect and process overlaps
for (int i = 0; i < appElemLocs.Count - 1; i++)
{
for (int j = i + 1; j < appElemLocs.Count; j++)
{
if (appElemLocs[i].Overlaps(appElemLocs[j]))
{
// pick the target between the two overlapping app's
AppElemLocations target, source;
int targetIndex, sourceIndex;
if (IsFirstTarget(appElemLocs[i], appElemLocs[j]))
{
target = appElemLocs[targetIndex = i];
source = appElemLocs[sourceIndex = j];
}
else
{
source = appElemLocs[sourceIndex = i];
target = appElemLocs[targetIndex = j];
}
Log.Logger.Information("Merging overlapping app " +
$"{GetAttributesDump(source.Element)} into " +
GetAttributesDump(target.Element));
// log error if the source had @wit/@source
if (LemHasLostAttributes(
source.Element.Element(XmlHelper.TEI + "lem"),
target.Element.Element(XmlHelper.TEI + "lem")))
{
string divId = source.Element.Ancestors(
XmlHelper.TEI + "div1")
.First()
.Attribute(XmlHelper.XML + "id").Value;
errDivIds.Add(Tuple.Create(docId, divId));
Log.Logger.Error("Removed overlapping app lost sources at div "
+ divId
+ ": "
+ GetAttributesDump(source.Element));
}
// append content of source into target in XML,
// excluding the lem child, and adding @n to each child
string nValue =
source.Element.Attribute("from").Value.Substring(1)
+ " "
+ source.Element.Attribute("to").Value.Substring(1);
foreach (XElement child in source.Element.Elements()
.Where(e => e.Name.LocalName != "lem"))
{
child.SetAttributeValue("n", nValue);
target.Element.Add(child);
}
// remove source from XML and locs
source.Element.Remove();
appElemLocs.RemoveAt(sourceIndex);
removedCount++;
// continue looking from overlaps from the first
// of the two app's involved
i = Math.Min(sourceIndex, targetIndex) - 1;
goto nextOuter;
}
} // j
nextOuter:
if (i % 10 == 0)
{
Console.Write('.');
}
} // i
// save
string path = Path.Combine(_outputDir, Path.GetFileName(filePath));
doc.Save(path, SaveOptions.OmitDuplicateNamespaces);
}
if (_writeDivList)
{
using (StreamWriter listWriter = new StreamWriter(
Path.Combine(_outputDir, "~overlap-err-divs.txt"),
false, Encoding.UTF8))
{
foreach (var id in errDivIds)
{
listWriter.WriteLine($"{id.Item1} {id.Item2}");
}
listWriter.Flush();
}
}
Console.WriteLine($"\nInput documents: {inputFileCount}");
Console.WriteLine($"Removed overlaps: {removedCount}");
return(Task.CompletedTask);
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("PARSE TEXT\n");
Console.ResetColor();
Console.WriteLine(
$"Input dir: {_inputFileDir}\n" +
$"Input mask: {_inputFileMask}\n" +
$"Output dir: {_outputDir}\n" +
$"Div IDs list: {_flagDivIdList ?? "(none)"}\n" +
$"Max items per file: {_maxItemPerFile}\n");
ILoggerFactory loggerFactory = new LoggerFactory();
loggerFactory.AddSerilog(Log.Logger);
Log.Logger.Information("PARSE TEXT");
XmlTextParser parser = new XmlTextParser
{
Logger = loggerFactory.CreateLogger("parse-text")
};
int inputFileCount = 0;
int totalItemCount = 0;
StreamWriter writer = null;
if (!Directory.Exists(_outputDir))
{
Directory.CreateDirectory(_outputDir);
}
// load div IDs list if requested, prefixing and suffixing them
// so that we are ready to find them in the item's title
HashSet <string> flagDivIds = _flagDivIdList != null
? LoadDivIds(_flagDivIdList, "xml:id=", XmlHelper.CIT_SEPARATOR)
: null;
// for each input document
foreach (string filePath in FileEnumerator.Enumerate(
_inputFileDir, _inputFileMask, _regexMask))
{
// load document
string inputFileName = Path.GetFileNameWithoutExtension(filePath);
Console.WriteLine("\n" + filePath);
inputFileCount++;
XDocument doc = XDocument.Load(filePath,
LoadOptions.PreserveWhitespace);
JsonSerializerSettings jsonSettings = new JsonSerializerSettings
{
ContractResolver = new DefaultContractResolver
{
NamingStrategy = new CamelCaseNamingStrategy()
},
Formatting = Formatting.Indented
};
// parse items
int itemCount = 0, outputFileCount = 0;
foreach (IItem item in parser.Parse(
doc, Path.GetFileNameWithoutExtension(filePath)))
{
if (++itemCount % 10 == 0)
{
Console.Write('.');
}
// set flag if required
if (flagDivIds.Any(s =>
item.Title.IndexOf(s, StringComparison.Ordinal) > -1))
{
item.Flags |= 1;
}
// create new output file if required
if (writer == null ||
(_maxItemPerFile > 0 && itemCount > _maxItemPerFile))
{
if (writer != null)
{
CloseOutputFile(writer);
}
string path = Path.Combine(_outputDir,
$"{inputFileName}_{++outputFileCount:00000}.json");
writer = new StreamWriter(new FileStream(path,
FileMode.Create, FileAccess.Write, FileShare.Read),
Encoding.UTF8);
writer.WriteLine("[");
}
// dump item into it
string json = JsonConvert.SerializeObject(
item, jsonSettings);
// string json = JsonSerializer.Serialize(item, typeof(object), options);
// this will output a , also for the last JSON array item,
// but we don't care about it -- that's just a dump, and
// it's easy to ignore/remove it if needed.
writer.WriteLine(json + ",");
}
totalItemCount += itemCount;
if (writer != null)
{
CloseOutputFile(writer);
writer = null;
}
}
Console.WriteLine($"\nInput documents: {inputFileCount}");
Console.WriteLine($"Output items: {totalItemCount}");
return(Task.CompletedTask);
}
/// <summary>
/// Goes through the Directories to scan recursively and starts a scan of each while that matches the whitelist. (both from settings)
/// </summary>
/// <returns>All found nuggets.</returns>
public IDictionary<string, TemplateItem> ParseAll()
{
IEnumerable<string> fileWhiteList = _settings.WhiteList;
IEnumerable<string> directoriesToSearchRecursively = _settings.DirectoriesToScan;
FileEnumerator fileEnumerator = new FileEnumerator(_settings.BlackList);
string currentFullPath;
bool blacklistFound = false;
var templateItems = new ConcurrentDictionary<string, TemplateItem>();
// Collection of template items keyed by their id.
foreach (var directoryPath in directoriesToSearchRecursively)
{
foreach (string filePath in fileEnumerator.GetFiles(directoryPath))
{
if (filePath.Length >= 260)
{
Console.WriteLine("Path too long to process. Path: " + filePath);
continue;
}
blacklistFound = false;
currentFullPath = Path.GetDirectoryName(Path.GetFullPath(filePath));
foreach (var blackItem in _settings.BlackList)
{
if (currentFullPath == null || currentFullPath.StartsWith(blackItem, StringComparison.OrdinalIgnoreCase))
{
//this is a file that is under a blacklisted directory so we do not parse it.
blacklistFound = true;
break;
}
}
if (!blacklistFound)
{
//we check every filePath against our white list. if it's on there in at least one form we check it.
foreach (var whiteListItem in fileWhiteList)
{
//We have a catch all for a filetype
if (whiteListItem.StartsWith("*."))
{
if (Path.GetExtension(filePath) == whiteListItem.Substring(1))
{
//we got a match
ParseFile(_settings.ProjectDirectory, filePath, templateItems);
break;
}
}
else //a file, like myfile.js
{
if (Path.GetFileName(filePath) == whiteListItem)
{
//we got a match
ParseFile(_settings.ProjectDirectory, filePath, templateItems);
break;
}
}
}
}
}
}
return templateItems;
}
