Exemple #1
0
 private static void Main(string[] args)
 {
     var vifObjects = new VifObjects();
     var fileEnumerator = new FileEnumerator(Path, "*.vif");
     var builder = new VifObjectBuilder();
     foreach (var path in fileEnumerator.EnumerateFiles())
     {
         //builder.Build(File.ReadAllLines(path));
     }
 }
Exemple #2
0
        /*
         * Delete all temporary files from the codegen directory (e.g. source files, ...)
         */
        internal void RemoveOldTempFiles()
        {
            Debug.Trace("BuildResultCache", "Deleting old temporary files from " + _cacheDir);

            RemoveCodegenResourceDir();

            string codegen = _cacheDir + "\\";

            // Go through all the files in the codegen dir
            foreach (FileData fileData in FileEnumerator.Create(codegen))
            {
                // Skip directories
                if (fileData.IsDirectory)
                {
                    continue;
                }

                // If it has a known extension, skip it
                string ext = Path.GetExtension(fileData.Name);
                if (ext == ".dll" || ext == ".pdb" || ext == ".web" || ext == ".ccu" || ext == ".prof" || ext == preservationFileExtension)
                {
                    continue;
                }

                // .delete files need to be removed.
                if (ext != dotDelete)
                {
                    // Don't delete the temp file if it's named after a dll that's still around
                    // since it could still be useful for debugging.
                    // Note that we can't use GetFileNameWithoutExtension here because
                    // some of the files are named 5hvoxl6v.0.cs, and it would return
                    // 5hvoxl6v.0 instead of just 5hvoxl6v
                    int periodIndex = fileData.Name.LastIndexOf('.');
                    if (periodIndex > 0)
                    {
                        string baseName = fileData.Name.Substring(0, periodIndex);

                        int secondPeriodIndex = baseName.LastIndexOf('.');
                        if (secondPeriodIndex > 0)
                        {
                            baseName = baseName.Substring(0, secondPeriodIndex);
                        }

                        // Generated source files uses assemblyname as prefix so we should keep them.
                        if (FileUtil.FileExists(codegen + baseName + ".dll"))
                        {
                            continue;
                        }

                        // other generated files, such as .cmdline, .err and .out need to add the
                        // WebAssemblyNamePrefix, since they do not use the assembly name as prefix.
                        if (FileUtil.FileExists(codegen + BuildManager.WebAssemblyNamePrefix + baseName + ".dll"))
                        {
                            continue;
                        }
                    }
                }
                else
                {
                    // Additional logic for VSWhidbey 564168 / Visual Studio QFE 4710.
                    // Delete both original .dll and .delete if possible
                    DiskBuildResultCache.CheckAndRemoveDotDeleteFile(new FileInfo(fileData.FullName));
                    continue;
                }

                Debug.Trace("BuildResultCache", "Deleting old temporary files: " + fileData.FullName);
                try {
                    File.Delete(fileData.FullName);
                } catch { }
            }
        }
Exemple #3
0
 public BaseCommand()
 {
     _fileReader     = new FileReader();
     _fileWriter     = new FileWriter();
     _fileEnumerator = new FileEnumerator();
 }
        /// <summary>
        /// Runs this command.
        /// </summary>
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine("PARTITION\n");
            Console.ResetColor();
            Console.WriteLine(
                $"Input dir:  {_inputDir}\n" +
                $"Input mask: {_fileMask}\n" +
                $"Output dir: {_outputDir}\n" +
                $"Min: {_minTreshold}\n" +
                $"Max: {_maxTreshold}\n" +
                $"Recursive: {_recursive}\n");
            Log.Logger.Information("PARTITION");

            XmlPartitioner partitioner = new XmlPartitioner
            {
                MinTreshold = _minTreshold,
                MaxTreshold = _maxTreshold
            };

            int partitioned = 0, total = 0;

            if (!Directory.Exists(_outputDir))
            {
                Directory.CreateDirectory(_outputDir);
            }

            foreach (string filePath in FileEnumerator.Enumerate(
                         _inputDir, _fileMask, _regexMask, _recursive))
            {
                total++;
                Console.Write(filePath);

                XDocument doc = XDocument.Load(filePath,
                                               LoadOptions.PreserveWhitespace);

                bool touched = partitioner.Partition(doc,
                                                     Path.GetFileNameWithoutExtension(filePath));

                string outputPath =
                    Path.Combine(_outputDir, Path.GetFileName(filePath));

                if (touched)
                {
                    partitioned++;
                    Console.WriteLine($" => {outputPath}");
                    if (!Directory.Exists(_outputDir))
                    {
                        Directory.CreateDirectory(_outputDir);
                    }
                    doc.Save(outputPath, SaveOptions.OmitDuplicateNamespaces);
                }
                else
                {
                    File.Copy(filePath, outputPath);
                    Console.WriteLine();
                }
            }

            Console.WriteLine($"Total files: {total}");
            Console.WriteLine($"Partitioned files: {partitioned}");

            return(Task.CompletedTask);
        }
Exemple #5
0
        private void BeginScanning()
        {
            if (ProcessingToggle.CurrentState == ToggleState.Active)
            {
                btnSearch.Enabled = false;
                ProcessingToggle.SetState(ToggleState.Inactive);
            }
            else if (ProcessingToggle.CurrentState == ToggleState.Ready)
            {
                btnSearch.Enabled = false;
                ProcessingToggle.SetState(ToggleState.Active);

                bool   calculateEntropy = checkboxCalculateEntropy.Checked;
                string selectedFolder   = tbPath.Text;
                string searchPatterns   = tbSearchPatterns.Text;

                List <YaraFilter> yaraParameters = new List <YaraFilter>();
                if (checkBoxYaraRules.Checked)
                {
                    yaraParameters = currentYaraFilters.ToList();
                }

                IDataPersistenceLayer dataPersistenceLayer = null;
                if (radioPersistenceCSV.Checked)
                {
                    dataPersistenceLayer = new CsvDataPersistenceLayer(tbPersistenceParameter.Text);
                }
                else if (radioPersistenceSqlite.Checked)
                {
                    dataPersistenceLayer = new SqliteDataPersistenceLayer(tbPersistenceParameter.Text);
                }
                else if (radioPersistenceSqlServer.Checked)
                {
                    dataPersistenceLayer = new SqlDataPersistenceLayer(tbPersistenceParameter.Text);
                }

                FileEnumeratorParameters parameters =
                    new FileEnumeratorParameters(cancelToken,
                                                 Settings.FileEnumeration_DisableWorkerThread,
                                                 selectedFolder,
                                                 searchPatterns,
                                                 calculateEntropy,
                                                 yaraParameters,
                                                 dataPersistenceLayer,
                                                 Log.ToUI, Log.ToFile, ReportNumbers, Log.ExceptionMessage);

                enumerationStart = DateTime.Now;

                bool didThrow = false;
                try
                {
                    parameters.ThrowIfAnyParametersInvalid();
                }
                catch (Exception ex)
                {
                    didThrow = true;
                    MessageBox.Show(
                        ex.ToString().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(),
                        MsgBox_TitleBarText,
                        MessageBoxButtons.OK,
                        MessageBoxIcon.Error
                        );
                }

                if (didThrow)
                {
                    ProcessingToggle.SetState(ToggleState.Ready);
                }
                else
                {
                    Log.ToUI(Environment.NewLine);
                    Log.ToAll($"Beginning Enumeration of folder: \"{parameters.SelectedFolder}\"");
                    Log.ToAll("Parsing MFT. (This may take a few minutes)");
                    FileEnumerator.LaunchFileEnumerator(parameters);
                }
            }
        }
Exemple #6
0
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine("IMPORT JSON TEXT AND APPARATUS\n");
            Console.ResetColor();
            Console.WriteLine(
                $"Text dir:  {_txtFileDir}\n" +
                $"Text mask: {_txtFileMask}\n" +
                $"Apparatus dir: {_appFileDir}\n" +
                $"Profile file: {_profilePath}\n" +
                $"Database: {_database}\n" +
                $"Dry run: {_dry}\n");

            ILoggerFactory loggerFactory = new LoggerFactory();

            loggerFactory.AddSerilog(Log.Logger);
            Log.Logger.Information("IMPORT JSON TEXT AND APPARATUS");

            if (!_dry)
            {
                // create database if not exists
                string connection = string.Format(CultureInfo.InvariantCulture,
                                                  _config.GetConnectionString("Mongo"),
                                                  _database);

                IDatabaseManager manager = new MongoDatabaseManager();

                string profileContent             = LoadProfile(_profilePath);
                IDataProfileSerializer serializer = new JsonDataProfileSerializer();
                DataProfile            profile    = serializer.Read(profileContent);

                if (!manager.DatabaseExists(connection))
                {
                    Console.WriteLine("Creating database...");
                    Log.Information($"Creating database {_database}...");

                    manager.CreateDatabase(connection, profile);

                    Console.WriteLine("Database created.");
                    Log.Information("Database created.");
                }
            }
            else
            {
                if (!File.Exists(_profilePath))
                {
                    string error = "Profile path not found: " + _profilePath;
                    Console.WriteLine(error);
                    Log.Error(error);
                    return(Task.CompletedTask);
                }
            }

            ICadmusRepository repository =
                _repositoryService.CreateRepository(_database);

            JsonImporter importer = new JsonImporter(repository)
            {
                Logger = loggerFactory.CreateLogger("json-importer"),
                IsDry  = _dry
            };

            int inputFileCount = 0;

            // 1) import text
            string[] files = FileEnumerator.Enumerate(
                _txtFileDir, _txtFileMask, _regexMask).ToArray();
            HashSet <string> fileNames = new HashSet <string>();

            Console.WriteLine($"Importing text from {files.Length} file(s)...");

            foreach (string txtFilePath in files)
            {
                fileNames.Add(
                    StripFileNameNr(
                        Path.GetFileNameWithoutExtension(txtFilePath)));
                Console.WriteLine(txtFilePath);
                inputFileCount++;

                using (Stream stream = new FileStream(txtFilePath, FileMode.Open,
                                                      FileAccess.Read, FileShare.Read))
                {
                    importer.ImportText(stream);
                }
            }

            // 2) import apparatus
            Console.WriteLine("Importing apparatus...");

            foreach (string fileName in fileNames)
            {
                Console.WriteLine(fileName);

                foreach (string appFilePath in Directory.EnumerateFiles(
                             _appFileDir, fileName + "-app_*.json"))
                {
                    Console.WriteLine("  " + appFilePath);
                    using (Stream stream = new FileStream(appFilePath, FileMode.Open,
                                                          FileAccess.Read, FileShare.Read))
                    {
                        importer.ImportApparatus(stream);
                    }
                }
            }

            return(Task.CompletedTask);
        }
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine("PARSE APPARATUS\n");
            Console.ResetColor();
            Console.WriteLine(
                $"Input:  {_inputFileMask}\n" +
                $"Output: {_outputDir}\n" +
                $"Max items per file: {_maxItemPerFile}\n");

            ILoggerFactory loggerFactory = new LoggerFactory();

            loggerFactory.AddSerilog(Log.Logger);
            Log.Logger.Information("PARSE APPARATUS");

            XmlApparatusParser parser = new XmlApparatusParser
            {
                Logger = loggerFactory.CreateLogger("parse-app")
            };

            int          inputFileCount = 0;
            int          totalPartCount = 0;
            StreamWriter writer         = null;

            if (!Directory.Exists(_outputDir))
            {
                Directory.CreateDirectory(_outputDir);
            }

            // for each input document
            foreach (string filePath in FileEnumerator.Enumerate(
                         _inputFileDir, _inputFileMask, _regexMask, _recursive))
            {
                Console.WriteLine();
                Log.Logger.Information("Parsing {FilePath}", filePath);

                // load document
                string inputFileName = Path.GetFileNameWithoutExtension(filePath);
                Console.WriteLine(filePath);
                inputFileCount++;
                XDocument doc = XDocument.Load(filePath,
                                               LoadOptions.PreserveWhitespace | LoadOptions.SetLineInfo);
                JsonSerializerSettings jsonSettings = new JsonSerializerSettings
                {
                    ContractResolver = new DefaultContractResolver
                    {
                        NamingStrategy = new CamelCaseNamingStrategy()
                    },
                    Formatting = Formatting.Indented
                };

                // load index
                string textFileName = inputFileName.Replace("-app", "");
                LoadTextIndex(textFileName);

                // parse
                int partCount = 0, outputFileCount = 0;

                foreach (var part in parser.Parse(doc, textFileName, _textIndex))
                {
                    if (++partCount % 10 == 0)
                    {
                        Console.Write('.');
                    }

                    // create new output file if required
                    if (writer == null ||
                        (_maxItemPerFile > 0 && partCount > _maxItemPerFile))
                    {
                        if (writer != null)
                        {
                            CloseOutputFile(writer);
                        }
                        string path = Path.Combine(_outputDir,
                                                   $"{inputFileName}_{++outputFileCount:00000}.json");

                        writer = new StreamWriter(new FileStream(path,
                                                                 FileMode.Create, FileAccess.Write, FileShare.Read),
                                                  Encoding.UTF8);
                        writer.WriteLine("[");
                    }

                    // dump part into it
                    string json = JsonConvert.SerializeObject(part, jsonSettings);
                    writer.WriteLine(json + ",");
                }
                totalPartCount += partCount;
                if (writer != null)
                {
                    CloseOutputFile(writer);
                    writer = null;
                }
            }

            Console.WriteLine($"\nInput documents: {inputFileCount}");
            Console.WriteLine($"Output parts: {totalPartCount}");

            return(Task.CompletedTask);
        }
Exemple #8
0
        /// <summary>
        /// Goes through the Directories to scan recursively and starts a scan of each while that matches the whitelist. (both from settings)
        /// </summary>
        /// <returns>All found nuggets.</returns>
        public IDictionary <string, TemplateItem> ParseAll()
        {
            IEnumerable <string> fileWhiteList = _settings.WhiteList;
            IEnumerable <string> directoriesToSearchRecursively = _settings.DirectoriesToScan;
            FileEnumerator       fileEnumerator = new FileEnumerator(_settings.BlackList);

            string currentFullPath;
            bool   blacklistFound = false;

            var templateItems = new ConcurrentDictionary <string, TemplateItem>();

            // Collection of template items keyed by their id.

            foreach (var directoryPath in directoriesToSearchRecursively)
            {
                foreach (string filePath in fileEnumerator.GetFiles(directoryPath))
                {
                    if (filePath.Length >= 260)
                    {
                        DebugHelpers.WriteLine("Path too long to process. Path: " + filePath);
                        continue;
                    }

                    blacklistFound  = false;
                    currentFullPath = Path.GetDirectoryName(Path.GetFullPath(filePath));
                    foreach (var blackItem in _settings.BlackList)
                    {
                        if (currentFullPath == null || currentFullPath.StartsWith(blackItem, StringComparison.OrdinalIgnoreCase))
                        {
                            //this is a file that is under a blacklisted directory so we do not parse it.
                            blacklistFound = true;
                            break;
                        }
                    }
                    if (!blacklistFound)
                    {
                        //we check every filePath against our white list. if it's on there in at least one form we check it.
                        foreach (var whiteListItem in fileWhiteList)
                        {
                            //We have a catch all for a filetype
                            if (whiteListItem.StartsWith("*."))
                            {
                                if (Path.GetExtension(filePath) == whiteListItem.Substring(1))
                                {
                                    //we got a match
                                    ParseFile(_settings.ProjectDirectory, filePath, templateItems);
                                    break;
                                }
                            }
                            else //a file, like myfile.js
                            {
                                if (Path.GetFileName(filePath) == whiteListItem)
                                {
                                    //we got a match
                                    ParseFile(_settings.ProjectDirectory, filePath, templateItems);
                                    break;
                                }
                            }
                        }
                    }
                }
            }

            return(templateItems);
        }
        internal MapPathBasedVirtualPathEnumerator(VirtualPath virtualPath, RequestedEntryType requestedEntryType)
        {
            if (virtualPath.IsRelative)
            {
                throw new ArgumentException(SR.GetString(SR.Invalid_app_VirtualPath), "virtualPath");
            }

            _virtualPath        = virtualPath;
            _requestedEntryType = requestedEntryType;

            string physicalPath;

            if (!ServerConfig.UseServerConfig)
            {
                // Use the hosting environment to map the virtual path
                physicalPath = _virtualPath.MapPathInternal();
            }
            else
            {
                IServerConfig serverConfig = ServerConfig.GetInstance();
                _serverConfig2 = serverConfig as IServerConfig2;

                // Use serverConfig to map the virtual path
                physicalPath = serverConfig.MapPath(null, _virtualPath);

                if (_requestedEntryType != RequestedEntryType.Files)
                {
                    // For MetabaseServerConfig, get the subdirs that are not in the application, and add them to the exclude list.
                    if (_serverConfig2 == null)
                    {
                        string [] virtualSubdirsNotInApp = serverConfig.GetVirtualSubdirs(_virtualPath, false);
                        if (virtualSubdirsNotInApp != null)
                        {
                            _exclude = new Hashtable(StringComparer.OrdinalIgnoreCase);
                            foreach (string subdir in virtualSubdirsNotInApp)
                            {
                                _exclude[subdir] = subdir;
                            }
                        }
                    }

                    // Get subdirs that are virtual directories, and record their physical mappings.
                    // Ignore the virtualPaths if we only need files, since it only contains directories
                    string [] virtualSubdirsInApp = serverConfig.GetVirtualSubdirs(_virtualPath, true);
                    if (virtualSubdirsInApp != null)
                    {
                        _virtualPaths = new Hashtable(StringComparer.OrdinalIgnoreCase);
                        foreach (string subdir in virtualSubdirsInApp)
                        {
                            VirtualPath subpath         = _virtualPath.SimpleCombineWithDir(subdir);
                            string      subPhysicalPath = serverConfig.MapPath(null, subpath);
                            if (FileUtil.DirectoryExists(subPhysicalPath))
                            {
                                _virtualPaths[subdir] = new MapPathBasedVirtualDirectory(subpath.VirtualPathString);
                            }
                        }

                        // Create enumerator for the virtual paths
                        _virtualEnumerator = _virtualPaths.Values.GetEnumerator();
                    }
                }
            }

            // Create an enumerator for the physical files and directories at this path
            _fileEnumerator = FileEnumerator.Create(physicalPath);

            // Reset the enumerator. Note that we don't support the Reset method.
            _useFileEnumerator = false;
        }
Exemple #10
0
 public IEnumerator GetEnumerator()
 {
     return(fileEnumerator = new FileEnumerator(this.filepath));
 }
Exemple #11
0
        /// <summary>
        /// Creates a <see cref="BackgroundWorker"/> class instance to update the song locations with a user selected path.
        /// </summary>
        /// <param name="folderBrowserDialog">An instance to <see cref="Ookii.Dialogs.WinForms.VistaFolderBrowserDialog"/> for the user to select the path.</param>
        /// <param name="connection">A SQLite connection to use for the procedure.</param>
        /// <returns>An instance to a <see cref="BackgroundWorker"/> class to handle the operation.</returns>
        public static BackgroundWorker UpdateSongLocations(
            VistaFolderBrowserDialog folderBrowserDialog, SQLiteConnection connection)
        {
            if (folderBrowserDialog.ShowDialog() == DialogResult.OK)
            {
                BackgroundWorker worker = new BackgroundWorker
                {
                    WorkerReportsProgress = true, WorkerSupportsCancellation = true
                };

                worker.DoWork += (sender, args) =>
                {
                    List <FileEnumeratorFileEntry> files = new List <FileEnumeratorFileEntry>(FileEnumerator
                                                                                              .RecurseFiles(folderBrowserDialog.SelectedPath,
                                                                                                            Constants.Extensions.ToArray()).ToArray());

                    if (files.Count == 0)
                    {
                        worker.ReportProgress(100);
                        return;
                    }

                    int fileCount = files.Count;

                    long songCount = GetScalar <long>(Song.GenerateCountSentence(false), connection);

                    int progress = 0;

                    int previousProgress = -1;

                    List <Song> songs = new List <Song>();
                    using (SQLiteCommand command = new SQLiteCommand(Song.GenerateSelectSentence(false), connection))
                    {
                        using (SQLiteDataReader reader = command.ExecuteReader())
                        {
                            while (reader.Read())
                            {
                                var percentage = progress * 50 / (int)songCount;

                                if (percentage > previousProgress)
                                {
                                    worker.ReportProgress(percentage);
                                    previousProgress = percentage;
                                }

                                songs.Add(Song.FromSqLiteDataReader(reader));
                                progress++;

                                if (worker.CancellationPending)
                                {
                                    return;
                                }
                            }
                        }
                    }

                    progress = 0;
                    foreach (var file in files)
                    {
                        try
                        {
                            var fileInfo = new FileInfo(file.FileNameWithPath);
                            var song     = songs.FirstOrDefault(f =>
                                                                f.FileSize == fileInfo.Length && f.FileNameNoPath == fileInfo.Name);

                            if (song == null)
                            {
                                song = songs.FirstOrDefault(f => f.Filename == fileInfo.FullName);
                                if (song != null)
                                {
                                    song.FileSize = (int)fileInfo.Length;
                                }
                                else
                                {
                                    song = songs.FirstOrDefault(f => f.FileNameNoPath == fileInfo.Name);
                                    if (song != null)
                                    {
                                        song.FileSize = (int)fileInfo.Length;
                                    }
                                }
                            }

                            var percentage = 50 + progress * 50 / fileCount;

                            if (percentage > previousProgress)
                            {
                                worker.ReportProgress(percentage);
                                previousProgress = percentage;
                            }

                            if (song == null)
                            {
                                continue;
                            }

                            song.Filename = file.FileNameWithPath;
                            ExecuteSql(song.GenerateInsertUpdateSqlSentence(false), connection);
                            progress++;

                            if (worker.CancellationPending)
                            {
                                return;
                            }
                        }
                        catch (Exception ex)
                        {
                            // log the exception..
                            ExceptionLogger.LogError(ex);
                        }
                    }

                    worker.ReportProgress(100);
                };

                return(worker);
            }

            return(null);
        }
        private static void Main(string[] args)
        {
            string connectionString = Settings.Database_ConnectionString;

            if (string.IsNullOrWhiteSpace(connectionString) || connectionString == "SetMe")
            {
                ReportOutput("ERROR: Connection string not set! Please set the SQL connection string in .config file.");
                ReportOutput("Aborting...");
                return;
            }
            else
            {
                FilePropertiesAccessLayer.SetConnectionString(connectionString);
            }

            if (args.Length == 0)
            {
                DisplayUsageSyntax();
                return;
            }

            // Will hold flag & parameter to flag, such as: "-p", "C:\Windows\"
            List <Tuple <string, string> > flags = GetFlags(args);

            if (!flags.Any())
            {
                return;
            }

            string searchPath       = "";
            string searchMask       = "*.*";
            bool   calcEntropy      = false;
            bool   onlineValidation = false;
            string yaraRulesFile    = "";

            foreach (Tuple <string, string> flagTuple in flags)
            {
                string flag      = flagTuple.Item1;
                string parameter = flagTuple.Item2;

                switch (flag)
                {
                case "e":
                    calcEntropy = true;
                    break;

                case "v":
                    onlineValidation = true;
                    break;

                case "p":
                    searchPath = parameter;
                    break;

                case "m":
                    searchMask = parameter;
                    break;

                case "y":
                    yaraRulesFile = parameter;
                    break;
                }
            }

            ReportOutput($"Search [P]ath: \"{searchPath}\"");
            ReportOutput($"Search [M]ask: {searchMask}");
            ReportOutput($"Calulate [E]ntropy: {calcEntropy}");
            ReportOutput($"Online [V]alidation: {onlineValidation}");
            ReportOutput($"[Y]ara Rules File: \"{yaraRulesFile}\"");
            ReportOutput("");

            FileEnumeratorParameters parameters = new FileEnumeratorParameters(CancellationToken.None, Settings.FileEnumeration_DisableWorkerThread, searchPath, searchMask, calcEntropy, onlineValidation, yaraRulesFile, ReportOutput, LogOutput, ReportResults, ReportException);

            ReportOutput("Beginning enumeration...");
            FileEnumerator.LaunchFileEnumerator(parameters);
        }
Exemple #13
0
 public void EnumerateFilesWithoutExtension_returns_empty_when_pass_in_null()
 {
     FileEnumerator.EnumerateFilesWithoutExtension(null).Should().BeEmpty();
 }
        /// <summary>
        /// Searches the TMDb database for a TV season based on a given path using a TMdbEasy.EasyClient class instance.
        /// </summary>
        /// <param name="easy">A TMdbEasy.EasyClient class instance to use for the search.</param>
        /// <param name="path">A path to enumerate files from.</param>
        /// <param name="stillSize">The size of the still image to get the URL for.</param>
        /// <param name="excludeFileNames">File names to be left out of the search.</param>
        /// <returns>A collection of TMDbDetail class instances containing the TV show season information from the TMDb database.</returns>
        public static async Task <IEnumerable <TMDbDetail> > GetSeasonAsync(EasyClient easy, string path, string stillSize = "original", List <string> excludeFileNames = null)
        {
            var televisionApi = easy.GetApi <ITelevisionApi>().Value; // create a ITelevisionApi..

            // avoid the null value..
            excludeFileNames = excludeFileNames ?? new List <string>();

            string fileName = string.Empty;

            if (File.Exists(path))
            {
                fileName = path;
                path     = Path.GetDirectoryName(path);
            }

            // List files of known video formats from the given path..
            IEnumerable <FileEnumeratorFileEntry> fileEntries =
                await FileEnumerator.RecurseFilesAsync(path, FileEnumerator.FiltersVideoVlcNoBinNoIso).ConfigureAwait(false);

            // remove the "useless" files from the list..
            List <FileEnumeratorFileEntry> tmpFilesList = new List <FileEnumeratorFileEntry>();

            // not in the include list..
            if (fileName != string.Empty)
            {
                fileEntries = fileEntries.Where(f => f.FileNameWithPath == fileName);
            }

            foreach (FileEnumeratorFileEntry entry in fileEntries.ToList())
            {
                // excluded from the list..
                if (excludeFileNames.Contains(entry.FileName))
                {
                    continue;            // ..so do continue..
                }
                tmpFilesList.Add(entry); // ..else add to the list..
            }

            fileEntries = tmpFilesList; // re-assign the list back to the source..

            // don't start searching if the directory is empty - we don't want to cause excess stress for the TMDb database..
            if (fileEntries.ToList().Count == 0)
            {
                // ..so just throw an exception..
                throw new Exception("No files were found from the given path string.");
            }

            // construct a search string of the given path..
            string searchString = GetTVShowSearchString(path);
            int    season       = GetTVShowSeason(path); // extract a season number from the given path..

            if (season == -1)                            // if no season number was in the given path..
            {
                // ..just throw an exception..
                throw new Exception("The TV season number wasn't found of the given path string.");
            }

            // initialize the result..
            List <TMDbDetail> result = new List <TMDbDetail>();

            // search for TV shows base on the search string build from the given directory name..
            TVShowList list = await televisionApi.SearchTVShowsAsync(searchString).ConfigureAwait(false);

            // if something was found..
            if (list != null && list.Total_results > 0)   // ..deepen the search..
            {
                string seriesName = list.Results[0].Name; // save the name of the TV show..
                int    seriesID   = list.Results[0].Id;   // save the ID of the TV show..

                // find the TV show season from the TMDb database with an ID and a season number..
                TvSeason tvSeason = await televisionApi.GetSeasonDetailsAsync(seriesID, season).ConfigureAwait(false);

                // if something was found..
                if (tvSeason != null && tvSeason.Episodes != null)
                {
                    foreach (Episode episode in tvSeason.Episodes) // return the details of the TV show season..
                    {
                        // don't return a file-less TMDbDetail class instance..
                        if (GetFileNameMatchingTVSeasonEpisode(fileEntries, season, episode.Episode_number) == null)
                        {
                            continue; // ..so just continue the loop..
                        }

                        result.Add(new TMDbDetail
                        {
                            ID        = seriesID,    // the TMDb id for the TV show..
                            SeasonID  = tvSeason.Id, // the TMDb id for the TV show season..
                            EpisodeID = episode.Id,  // the TMDb id for the TV show season episode..

                            // formulate the title base on the TVEpisodeFormat or the "overriding" TVEpisodeFormatCustom property value..
                            Title = TVEpisodeFormatCustom != string.Empty ?
                                    TVEpisodeFormatCustom.
                                    Replace("#SNUM#", tvSeason.Season_number.ToString()).                 // the season number as one digit..
                                    Replace("#ENUM#", episode.Episode_number.ToString()).                 // the episode number as one digit..
                                    Replace("#EPISODE_NAME#", episode.Name).                              // the name of the episode..
                                    Replace("#SERIES_NAME#", seriesName).                                 // the name of the series..
                                    Replace("#SEASON_NAME#", tvSeason.Name).                              // the name of the TV show season..
                                    Replace("#SNUM2#", string.Format("{0:00}", tvSeason.Season_number)).  // the season number as two digits..
                                    Replace("#ENUM2#", string.Format("{0:00}", episode.Episode_number)) : // the episode number as two digits..
                                    string.Format(TVEpisodeFormat, seriesName, tvSeason.Name, episode.Episode_number, episode.Name),

                            // set the description..
                            Description       = string.IsNullOrEmpty(tvSeason.Overview) ? episode.Overview : tvSeason.Overview,
                            DetailDescription = episode.Overview, // set the detailed description if any..

                            // find the file name for the TV show episode..
                            FileName = GetFileNameMatchingTVSeasonEpisode(fileEntries, season, episode.Episode_number).FileName,

                            // create an URL for the still using the TV season's poster path as a "fail safe"..
                            PosterOrStillURL = new Uri("https://image.tmdb.org/t/p/" + stillSize +
                                                       (string.IsNullOrEmpty(episode.Still_path) ? tvSeason.Poster_path : episode.Still_path)),
                            Season  = season,                // set the season number..
                            Episode = episode.Episode_number // set the episode number..
                        });
                    }
                }
                else // nothing was found..
                {
                    // loop through the found files..
                    foreach (FileEnumeratorFileEntry entry in fileEntries)
                    {
                        result.Add(new TMDbDetail
                        {
                            Title    = entry.FileNameNoExtension,             // the title can be the file name without path or extension..
                            FileName = entry.FileName,                        // set the file name..
                            Season   = season,                                // set the season number..
                            Episode  = GetTVShowEpisodeNumber(entry.FileName) // set the episode number..
                        });
                    }
                }
            }
            return(result);
        }
        /// <summary>
        /// Upload a directory to Azure File Storage.
        /// </summary>
        /// <param name="sourcePath">Path to the source directory</param>
        /// <param name="destFileDir">The <see cref="CloudFileDirectory"/> that is the destination Azure file directory.</param>
        /// <param name="options">An <see cref="UploadDirectoryOptions"/> object that specifies additional options for the operation.</param>
        /// <param name="context">A <see cref="TransferContext"/> object that represents the context for the current operation.</param>
        /// <param name="cancellationToken">A <see cref="CancellationToken"/> object to observe while waiting for a task to complete.</param>
        /// <returns>A <see cref="Task"/> object that represents the asynchronous operation.</returns>
        public static Task UploadDirectoryAsync(string sourcePath, CloudFileDirectory destFileDir, UploadDirectoryOptions options, TransferContext context, CancellationToken cancellationToken)
        {
            DirectoryLocation sourceLocation = new DirectoryLocation(sourcePath);
            AzureFileDirectoryLocation destLocation = new AzureFileDirectoryLocation(destFileDir);
            FileEnumerator sourceEnumerator = new FileEnumerator(sourceLocation);
            if (options != null)
            {
                sourceEnumerator.SearchPattern = options.SearchPattern;
                sourceEnumerator.Recursive = options.Recursive;
            }

            return UploadDirectoryInternalAsync(sourceLocation, destLocation, sourceEnumerator, options, context, cancellationToken);
        }
        private static void Main(string[] args)
        {
            if (args.Length == 0)
            {
                DisplayUsageSyntax();
                return;
            }

            // Will hold flag & parameter to flag, such as: "-p", "C:\Windows\"
            List <Tuple <string, string> > flags = GetFlags(args);

            if (!flags.Any())
            {
                DisplayUsageSyntax();
                return;
            }

            string            searchPath          = "";
            string            searchMask          = "*.*";
            bool              isEntropyEnabled    = false;
            bool              isYaraEnabled       = false;
            bool              isSqlServerEnabled  = false;
            bool              isSqliteEnabled     = false;
            bool              isCsvEnabled        = false;
            string            sqliteDbFile        = "";
            string            csvFile             = "";
            string            sqlConnectionString = (Settings.Database_ConnectionString == "SetMe") ? "" : Settings.Database_ConnectionString;
            string            yaraFiltersFile     = "";
            List <YaraFilter> yaraFilters         = new List <YaraFilter>();

            foreach (Tuple <string, string> flagTuple in flags)
            {
                string flag      = flagTuple.Item1;
                string parameter = flagTuple.Item2;

                switch (flag)
                {
                case "e":
                    isEntropyEnabled = true;
                    break;

                case "p":
                    searchPath = parameter;
                    break;

                case "m":
                    searchMask = parameter;
                    break;

                case "y":
                    isYaraEnabled   = true;
                    yaraFiltersFile = parameter;
                    break;

                case "s":
                    isSqlServerEnabled = true;
                    break;

                case "l":
                    isSqliteEnabled = true;
                    sqliteDbFile    = string.IsNullOrWhiteSpace(parameter) ? sqlConnectionString : parameter;
                    break;

                case "c":
                    isCsvEnabled = true;
                    csvFile      = parameter;
                    break;
                }
            }

            ReportOutput();
            ReportOutput("Running with these parameters:");
            ReportOutput($"   Search [P]ath:       \"{searchPath}\"");
            ReportOutput($"   Search [M]ask:        {searchMask}");
            ReportOutput($"   Calulate [E]ntropy:   {isEntropyEnabled}");
            if (isYaraEnabled)
            {
                ReportOutput($"   [Y]ara filters file: \"{yaraFiltersFile}\"");
            }

            if (isSqlServerEnabled)
            {
                ReportOutput($"   [S]QL connection: \"{sqlConnectionString}\"");
            }
            else if (isSqliteEnabled)
            {
                ReportOutput($"   Sq[l]ite DB: \"{sqliteDbFile}\"");
            }
            else if (isCsvEnabled)
            {
                ReportOutput($"   [C]SV file: \"{csvFile}\"");
            }

            ReportOutput();

            if (string.IsNullOrWhiteSpace(searchPath))
            {
                ReportOutput("No search path provided!");
                ReportOutput("You must supply the -p flag with a path, e.g.:");
                ReportOutput("-p:\"C:\\Program Files\\BanzaiBuddy\"");
                ReportOutput();
                ReportOutput("Aborting...");
                return;
            }

            if (isYaraEnabled)
            {
                if (!File.Exists(yaraFiltersFile))
                {
                    ReportOutput($"The yara filters file path suppled does not exist: \"{yaraFiltersFile}\".");
                    ReportOutput();
                    ReportOutput("Aborting...");
                    return;
                }
                try
                {
                    string loadJson = File.ReadAllText(yaraFiltersFile);
                    yaraFilters = JsonConvert.DeserializeObject <List <YaraFilter> >(loadJson);
                }
                catch
                {
                    ReportOutput("The yara filters file must be a JSON file.");
                    ReportOutput();
                    ReportOutput("Aborting...");
                    return;
                }
            }

            IDataPersistenceLayer dataPersistenceLayer;

            if (isSqliteEnabled)
            {
                dataPersistenceLayer = new SqliteDataPersistenceLayer(sqliteDbFile);
            }
            else if (isCsvEnabled)
            {
                dataPersistenceLayer = new CsvDataPersistenceLayer(csvFile);
            }
            else if (isSqlServerEnabled)
            {
                dataPersistenceLayer = new SqlDataPersistenceLayer(sqlConnectionString);
            }
            else
            {
                ReportOutput("No output parameter provided!");

                if (!string.IsNullOrWhiteSpace(sqlConnectionString))
                {
                    ReportOutput("(SQL server connection string supplied in config file, asuming SQL server output...)");
                    dataPersistenceLayer = new SqlDataPersistenceLayer(sqlConnectionString);
                }
                else
                {
                    ReportOutput("You must supply an output parameter, e.g.:");
                    ReportOutput("-c:C:\\out.csv");
                    ReportOutput($"OR provide a SQL server connection string in the config file: {_thisExecutableFilename}");
                    ReportOutput("(Because it defaults to a SQL server connection. However, the connection string was missing.)");
                    ReportOutput();
                    ReportOutput("Aborting...");
                    return;
                }
            }

            FileEnumeratorParameters parameters =
                new FileEnumeratorParameters(
                    CancellationToken.None,
                    true,                             // Do not change this. If set to false, it will run on a thread, return immediately and exit, killing the thread.
                    searchPath,
                    searchMask,
                    isEntropyEnabled,
                    yaraFilters,
                    dataPersistenceLayer,
                    ReportOutput,
                    Log.LogOutputAction,
                    ReportResults,
                    Log.ExceptionMessage
                    );

            parameters.ThrowIfAnyParametersInvalid();

            ReportOutput("Beginning scan...");
            FileEnumerator.LaunchFileEnumerator(parameters);
        }
Exemple #17
0
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine("REPORT OVERLAPS\n");
            Console.ResetColor();
            Console.WriteLine(
                $"Input:  {_appFileMask}\n" +
                $"Output: {_outputPath}\n");

            int inputFileCount = 0;
            int overlapCount   = 0;

            ILoggerFactory loggerFactory = new LoggerFactory();

            loggerFactory.AddSerilog(Log.Logger);
            Log.Logger.Information("REPORT OVERLAPS");

            using (StreamWriter writer = new StreamWriter(_outputPath, false,
                                                          Encoding.UTF8))
            {
                writer.WriteLine("# Overlaps Report");
                writer.WriteLine();

                writer.WriteLine($"Input: `{_appFileDir}{Path.DirectorySeparatorChar}{_appFileMask}`");
                writer.WriteLine();

                // for each app document
                WordIdList widList = new WordIdList
                {
                    Logger = loggerFactory.CreateLogger("report-overlaps")
                };
                foreach (string filePath in FileEnumerator.Enumerate(
                             _appFileDir, _appFileMask, _regexMask, _recursive))
                {
                    Console.WriteLine();
                    Log.Logger.Information("Parsing {FilePath}", filePath);

                    // load app document
                    string inputFileName = Path.GetFileNameWithoutExtension(filePath);
                    Console.WriteLine(filePath);
                    inputFileCount++;
                    XDocument doc = XDocument.Load(filePath,
                                                   LoadOptions.PreserveWhitespace | LoadOptions.SetLineInfo);

                    // collect word IDs from text document
                    widList.Parse(XDocument.Load(filePath.Replace("-app.", ".")));

                    // collect app's locations
                    List <AppElemLocations> appElemLocs =
                        AppElemLocationCollector.Collect(doc, widList,
                                                         AppElemLocationCollector.IsOverlappable);

                    // detect and report overlaps
                    for (int i = 0; i < appElemLocs.Count - 1; i++)
                    {
                        for (int j = i + 1; j < appElemLocs.Count; j++)
                        {
                            if (appElemLocs[i].Overlaps(appElemLocs[j]))
                            {
                                writer.WriteLine($"## Overlap {++overlapCount}");
                                writer.WriteLine();
                                writer.WriteLine(Path.GetFileName(filePath) +
                                                 $" at {appElemLocs[i].LineNumber}");

                                // text
                                int n = 0;
                                foreach (var iw in appElemLocs[i].Locations)
                                {
                                    if (++n > 1)
                                    {
                                        writer.Write(' ');
                                    }
                                    writer.Write($"`{iw.Item1}`=`{iw.Item2}`");
                                }
                                writer.WriteLine();
                                writer.WriteLine();

                                // app
                                WriteAppXml(appElemLocs[i], writer);
                                WriteAppXml(appElemLocs[j], writer);
                                goto nextOuter;
                            }
                        }
nextOuter:
                        if (i % 10 == 0)
                        {
                            Console.Write('.');
                        }
                    }
                    Console.WriteLine();
                }
                writer.Flush();
            }

            Console.WriteLine($"\nInput documents: {inputFileCount}");
            return(Task.CompletedTask);
        }
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine("REMOVE OVERLAPS\n");
            Console.ResetColor();
            Console.WriteLine(
                $"Input:  {_appFileMask}\n" +
                $"Output: {_outputDir}\n" +
                $"Div list: {(_writeDivList ? "yes" : "no")}\n");

            int inputFileCount = 0;
            int removedCount   = 0;

            ILoggerFactory loggerFactory = new LoggerFactory();

            loggerFactory.AddSerilog(Log.Logger);
            Log.Logger.Information("REMOVE OVERLAPS");

            if (!Directory.Exists(_outputDir))
            {
                Directory.CreateDirectory(_outputDir);
            }

            HashSet <Tuple <string, string> > errDivIds =
                new HashSet <Tuple <string, string> >();

            // for each app document
            WordIdList widList = new WordIdList
            {
                Logger = loggerFactory.CreateLogger("report-overlaps")
            };

            foreach (string filePath in FileEnumerator.Enumerate(
                         _appFileDir, _appFileMask, _regexMask, _recursive))
            {
                Console.WriteLine();
                Log.Logger.Information("Parsing {FilePath}", filePath);
                string docId = Path.GetFileNameWithoutExtension(filePath)
                               .Replace("-app", "");

                // load app document
                string inputFileName = Path.GetFileNameWithoutExtension(filePath);
                Console.WriteLine(filePath);
                inputFileCount++;
                XDocument doc = XDocument.Load(filePath,
                                               LoadOptions.PreserveWhitespace | LoadOptions.SetLineInfo);

                // collect word IDs from text document
                widList.Parse(XDocument.Load(filePath.Replace("-app.", ".")));

                // collect app's locations
                List <AppElemLocations> appElemLocs =
                    AppElemLocationCollector.Collect(doc, widList,
                                                     AppElemLocationCollector.IsOverlappable);

                // detect and process overlaps
                for (int i = 0; i < appElemLocs.Count - 1; i++)
                {
                    for (int j = i + 1; j < appElemLocs.Count; j++)
                    {
                        if (appElemLocs[i].Overlaps(appElemLocs[j]))
                        {
                            // pick the target between the two overlapping app's
                            AppElemLocations target, source;
                            int targetIndex, sourceIndex;

                            if (IsFirstTarget(appElemLocs[i], appElemLocs[j]))
                            {
                                target = appElemLocs[targetIndex = i];
                                source = appElemLocs[sourceIndex = j];
                            }
                            else
                            {
                                source = appElemLocs[sourceIndex = i];
                                target = appElemLocs[targetIndex = j];
                            }

                            Log.Logger.Information("Merging overlapping app " +
                                                   $"{GetAttributesDump(source.Element)} into " +
                                                   GetAttributesDump(target.Element));

                            // log error if the source had @wit/@source
                            if (LemHasLostAttributes(
                                    source.Element.Element(XmlHelper.TEI + "lem"),
                                    target.Element.Element(XmlHelper.TEI + "lem")))
                            {
                                string divId = source.Element.Ancestors(
                                    XmlHelper.TEI + "div1")
                                               .First()
                                               .Attribute(XmlHelper.XML + "id").Value;

                                errDivIds.Add(Tuple.Create(docId, divId));
                                Log.Logger.Error("Removed overlapping app lost sources at div "
                                                 + divId
                                                 + ": "
                                                 + GetAttributesDump(source.Element));
                            }

                            // append content of source into target in XML,
                            // excluding the lem child, and adding @n to each child
                            string nValue =
                                source.Element.Attribute("from").Value.Substring(1)
                                + " "
                                + source.Element.Attribute("to").Value.Substring(1);
                            foreach (XElement child in source.Element.Elements()
                                     .Where(e => e.Name.LocalName != "lem"))
                            {
                                child.SetAttributeValue("n", nValue);
                                target.Element.Add(child);
                            }

                            // remove source from XML and locs
                            source.Element.Remove();
                            appElemLocs.RemoveAt(sourceIndex);
                            removedCount++;

                            // continue looking from overlaps from the first
                            // of the two app's involved
                            i = Math.Min(sourceIndex, targetIndex) - 1;
                            goto nextOuter;
                        }
                    } // j
nextOuter:
                    if (i % 10 == 0)
                    {
                        Console.Write('.');
                    }
                } // i

                // save
                string path = Path.Combine(_outputDir, Path.GetFileName(filePath));
                doc.Save(path, SaveOptions.OmitDuplicateNamespaces);
            }

            if (_writeDivList)
            {
                using (StreamWriter listWriter = new StreamWriter(
                           Path.Combine(_outputDir, "~overlap-err-divs.txt"),
                           false, Encoding.UTF8))
                {
                    foreach (var id in errDivIds)
                    {
                        listWriter.WriteLine($"{id.Item1} {id.Item2}");
                    }
                    listWriter.Flush();
                }
            }

            Console.WriteLine($"\nInput documents: {inputFileCount}");
            Console.WriteLine($"Removed overlaps: {removedCount}");
            return(Task.CompletedTask);
        }
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine("PARSE TEXT\n");
            Console.ResetColor();
            Console.WriteLine(
                $"Input dir:  {_inputFileDir}\n" +
                $"Input mask: {_inputFileMask}\n" +
                $"Output dir: {_outputDir}\n" +
                $"Div IDs list: {_flagDivIdList ?? "(none)"}\n" +
                $"Max items per file: {_maxItemPerFile}\n");

            ILoggerFactory loggerFactory = new LoggerFactory();

            loggerFactory.AddSerilog(Log.Logger);
            Log.Logger.Information("PARSE TEXT");

            XmlTextParser parser = new XmlTextParser
            {
                Logger = loggerFactory.CreateLogger("parse-text")
            };

            int          inputFileCount = 0;
            int          totalItemCount = 0;
            StreamWriter writer         = null;

            if (!Directory.Exists(_outputDir))
            {
                Directory.CreateDirectory(_outputDir);
            }

            // load div IDs list if requested, prefixing and suffixing them
            // so that we are ready to find them in the item's title
            HashSet <string> flagDivIds = _flagDivIdList != null
                ? LoadDivIds(_flagDivIdList, "xml:id=", XmlHelper.CIT_SEPARATOR)
                : null;

            // for each input document
            foreach (string filePath in FileEnumerator.Enumerate(
                         _inputFileDir, _inputFileMask, _regexMask))
            {
                // load document
                string inputFileName = Path.GetFileNameWithoutExtension(filePath);
                Console.WriteLine("\n" + filePath);
                inputFileCount++;
                XDocument doc = XDocument.Load(filePath,
                                               LoadOptions.PreserveWhitespace);
                JsonSerializerSettings jsonSettings = new JsonSerializerSettings
                {
                    ContractResolver = new DefaultContractResolver
                    {
                        NamingStrategy = new CamelCaseNamingStrategy()
                    },
                    Formatting = Formatting.Indented
                };

                // parse items
                int itemCount = 0, outputFileCount = 0;

                foreach (IItem item in parser.Parse(
                             doc, Path.GetFileNameWithoutExtension(filePath)))
                {
                    if (++itemCount % 10 == 0)
                    {
                        Console.Write('.');
                    }

                    // set flag if required
                    if (flagDivIds.Any(s =>
                                       item.Title.IndexOf(s, StringComparison.Ordinal) > -1))
                    {
                        item.Flags |= 1;
                    }

                    // create new output file if required
                    if (writer == null ||
                        (_maxItemPerFile > 0 && itemCount > _maxItemPerFile))
                    {
                        if (writer != null)
                        {
                            CloseOutputFile(writer);
                        }
                        string path = Path.Combine(_outputDir,
                                                   $"{inputFileName}_{++outputFileCount:00000}.json");

                        writer = new StreamWriter(new FileStream(path,
                                                                 FileMode.Create, FileAccess.Write, FileShare.Read),
                                                  Encoding.UTF8);
                        writer.WriteLine("[");
                    }

                    // dump item into it
                    string json = JsonConvert.SerializeObject(
                        item, jsonSettings);
                    // string json = JsonSerializer.Serialize(item, typeof(object), options);
                    // this will output a , also for the last JSON array item,
                    // but we don't care about it -- that's just a dump, and
                    // it's easy to ignore/remove it if needed.
                    writer.WriteLine(json + ",");
                }
                totalItemCount += itemCount;
                if (writer != null)
                {
                    CloseOutputFile(writer);
                    writer = null;
                }
            }

            Console.WriteLine($"\nInput documents: {inputFileCount}");
            Console.WriteLine($"Output items: {totalItemCount}");

            return(Task.CompletedTask);
        }
Exemple #20
0
		/// <summary>
		/// Goes through the Directories to scan recursively and starts a scan of each while that matches the whitelist. (both from settings)
		/// </summary>
		/// <returns>All found nuggets.</returns>
		public IDictionary<string, TemplateItem> ParseAll()
		{
			IEnumerable<string> fileWhiteList = _settings.WhiteList;
			IEnumerable<string> directoriesToSearchRecursively = _settings.DirectoriesToScan;
            FileEnumerator fileEnumerator = new FileEnumerator(_settings.BlackList);

			string currentFullPath;
			bool blacklistFound = false;

			var templateItems = new ConcurrentDictionary<string, TemplateItem>();
                // Collection of template items keyed by their id.

			foreach (var directoryPath in directoriesToSearchRecursively)
			{
                foreach (string filePath in fileEnumerator.GetFiles(directoryPath))
				{
                    if (filePath.Length >= 260)
                    {
                        Console.WriteLine("Path too long to process. Path: " + filePath);
                        continue;
                    }

					blacklistFound = false;
                    currentFullPath = Path.GetDirectoryName(Path.GetFullPath(filePath));
                    foreach (var blackItem in _settings.BlackList)
                    {
                        if (currentFullPath == null || currentFullPath.StartsWith(blackItem, StringComparison.OrdinalIgnoreCase))
                        {
                            //this is a file that is under a blacklisted directory so we do not parse it.
                            blacklistFound = true;
                            break;
                        }
                    }
                    if (!blacklistFound)
                    {


                        //we check every filePath against our white list. if it's on there in at least one form we check it.
                        foreach (var whiteListItem in fileWhiteList)
                        {
                            //We have a catch all for a filetype
                            if (whiteListItem.StartsWith("*."))
                            {
                                if (Path.GetExtension(filePath) == whiteListItem.Substring(1))
                                {
                                    //we got a match
                                    ParseFile(_settings.ProjectDirectory, filePath, templateItems);
                                    break;
                                }
                            }
                            else //a file, like myfile.js
                            {
                                if (Path.GetFileName(filePath) == whiteListItem)
                                {
                                    //we got a match
                                    ParseFile(_settings.ProjectDirectory, filePath, templateItems);
                                    break;
                                }
                            }
                        }

                    }
				}
			}

			return templateItems;
		}