private static void Main(string[] args)
{
var config = ConfigurationFactory.Build(args);
Console.Write("Searching for files...");
var parser = new FolderParser(new DirectoryEnumerator());
var sourceFiles = parser.Files(config.Input.Sources.First()).ToList();
Console.WriteLine("[Done]");
var extractor = BuildWordExtractor(config.Input);
var fileReader = new FileReader();
var words = new Dictionary <string, int>();
var sourceFileContents = sourceFiles.Select(sourceFile => fileReader.Read(sourceFile));
var sourceFileWords = sourceFileContents.AsParallel().Select(text => extractor.Extract(text));
Console.Write($"Extracting words from {sourceFiles.Count} source files...");
foreach (var wordsInFile in sourceFileWords)
{
words.Merge(wordsInFile);
}
Console.WriteLine("[Done]");
var baseName = Path.Combine(config.Output.TargetPath, "noiseMap");
GenerateCsv(config.Output.Csv, words, baseName);
GenerateImage(config.Output.Image, words, baseName, config.Output.Seed);
}
static void Main(string[] args)
{
string[] exceptedWords = { "I", "a", "i" };
string[] replaceWords = { " ", " ", " " };
List <ICleaner> cleaners = new List <ICleaner>();
cleaners.Add(new NumberRemover());
cleaners.Add(new SmallWordRemover(1, exceptedWords));
cleaners.Add(new WordReplacer(replaceWords));
//cleaners.Add(new SegmentSplitter());
cleaners.Add(new Trimmer());
cleaners.Add(new IdenticalRemover());
cleaners.Add(new DuplicateRemover());
FolderParser folderParser = new FolderParser(fileExtensions);
List <string> files = folderParser.parseFolder("G:\\Översättning\\en-US-sv-SE\\");
TSVLoader loader = new TSVLoader("\t");
Serializer serializer = new Serializer();
foreach (string file in files)
{
SentenceFile sentenceFile = loader.loadTSV(file);
foreach (ICleaner cleaner in cleaners)
{
sentenceFile.sentencePairs = cleaner.CleanSentencePairs(sentenceFile.sentencePairs);
}
serializer.serialize(sentenceFile);
}
}
public void ParseFoldersWhenMultipleFolderNodeAndCreateThem()
{
var mockFolderService = new Mock <FolderService>();
mockFolderService.Setup(s => s.Create(It.IsAny <string>(), It.IsAny <string>())).Verifiable();
var folderService = mockFolderService.Object;
var stubReportService = new Mock <ReportService>();
stubReportService.Setup(s => s.Create(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <Dictionary <string, string> >())).Verifiable();
var reportService = stubReportService.Object;
var stubReportParser = new Mock <ReportParser>(reportService);
stubReportParser.Setup(s => s.Execute(It.IsAny <XmlNode>())).Verifiable();
var childParser = (IParser)stubReportParser.Object;
var folderParser = new FolderParser(folderService, Enumerable.Repeat(childParser, 1));
var xmlDoc = new XmlDocument();
using (Stream stream = Assembly.GetExecutingAssembly()
.GetManifestResourceStream("RsPackage.Testing.Resources.MultiLevelSample.xml"))
using (StreamReader reader = new StreamReader(stream))
xmlDoc.Load(reader);
var root = xmlDoc.FirstChild.NextSibling;
folderParser.Execute(root);
Mock.Get(folderService).Verify(s => s.Create("Analysis", null));
Mock.Get(folderService).Verify(s => s.Create("Low-level", null));
Mock.Get(folderService).Verify(s => s.Create("Realtime", "/Analysis"));
Mock.Get(folderService).Verify(s => s.Create("Confidential", "/Analysis/Realtime"));
}
public void ParseFolderWithPolicy()
{
var stubFolderService = new Mock <FolderService>();
stubFolderService.Setup(s => s.Create(It.IsAny <string>(), It.IsAny <string>()));
var folderService = stubFolderService.Object;
var mockPolicyParser = new Mock <PolicyParser>();
mockPolicyParser.Setup(p => p.Execute(It.IsAny <XmlNode>())).Verifiable();
var policyParser = mockPolicyParser.Object;
var parser = new FolderParser(folderService, new[] { policyParser });
var xmlDoc = new XmlDocument();
using (Stream stream = Assembly.GetExecutingAssembly()
.GetManifestResourceStream("RsPackage.Testing.Resources.MultiLevelSample.xml"))
using (StreamReader reader = new StreamReader(stream))
xmlDoc.Load(reader);
var root = xmlDoc.FirstChild.NextSibling.SelectSingleNode("./Folder[@Name='Analysis']");
parser.Execute(root);
Mock.Get(policyParser).Verify(p => p.Execute(It.IsAny <XmlNode>()), Times.Exactly(2));
}
public ProjectParser GetXmlParser(Options options)
{
var serviceBuilder = new ServiceBuilder();
serviceBuilder.Setup(options);
serviceBuilder.Build();
var rootPath = GetRootPath(options);
var parentFolder = GetParentFolder(options);
var namingConvention = GetNamingConvention(options);
var parser = new ProjectParser()
{
ParentFolder = parentFolder,
RootPath = rootPath,
NamingConvention = namingConvention
};
var policyParser = new PolicyParser(serviceBuilder.GetPolicyService());
var dataSourceParser = new DataSourceParser(serviceBuilder.GetDataSourceService());
var reportParser = new ReportParser(serviceBuilder.GetReportService(), new[] { policyParser });
var folderParser = new FolderParser(serviceBuilder.GetFolderService(), new IParser[] { policyParser, dataSourceParser, reportParser });
parser.ChildParsers.Add(dataSourceParser);
parser.ChildParsers.Add(reportParser);
parser.ChildParsers.Add(folderParser);
return(parser);
}
public void ParseReportsMultipleFolderNode()
{
var stubFolderService = new Mock <FolderService>();
stubFolderService.Setup(s => s.Create(It.IsAny <string>(), It.IsAny <string>())).Verifiable();
var folderService = stubFolderService.Object;
var stubReportService = new Mock <ReportService>();
stubReportService.Setup(s => s.Create(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <bool>(), It.IsAny <IDictionary <string, string> >(), It.IsAny <IDictionary <string, string> >())).Verifiable();
var reportService = stubReportService.Object;
var mockReportParser = new Mock <ReportParser>(reportService);
mockReportParser.Setup(s => s.Execute(It.IsAny <XmlNode>())).Verifiable();
var childParser = (IParser)mockReportParser.Object;
var folderParser = new FolderParser(folderService, Enumerable.Repeat(childParser, 1));
var xmlDoc = new XmlDocument();
using (Stream stream = Assembly.GetExecutingAssembly()
.GetManifestResourceStream("RsPackage.Testing.Resources.MultiLevelSample.xml"))
using (StreamReader reader = new StreamReader(stream))
xmlDoc.Load(reader);
var root = xmlDoc.FirstChild.NextSibling;
folderParser.Execute(root);
Mock.Get(childParser).Verify(s => s.Execute(It.IsAny <XmlNode>()), Times.Exactly(4));
}
public ParserProgram(string pattern)
{
Pattern = pattern;
Parser = new FolderParser();
RecipeParser = new RecipeParser();
BasicParser = new BasicParser();
Provider = new ServiceProvider();
}
private void ImportFolderToolStripMenuItem_Click(object sender, EventArgs e)
{
folderBrowserDlg.SelectedPath = PathManager.Init.GetLastPath(Settings.Default.ImportLastPath);
if (folderBrowserDlg.ShowFolderBrowser(this) == DialogResult.OK)
{
Settings.Default.ImportLastPath = folderBrowserDlg.SelectedPath;
var folderParser = new FolderParser();
ImportSourceFolder(folderParser, folderBrowserDlg.SelectedPath);
}
}
private void SourceFilesTree_DragDrop(object sender, DragEventArgs e)
{
var fileData = e.Data.GetData(TreeNode_Drop_FileDrop) as string[];
if (fileData != null)
{
var folderParser = new FolderParser();
foreach (var item in fileData)
{
ImportSourceFolder(folderParser, item);
}
}
}
public void ShouldContainAllFoldersInRootFolder()
{
// Arrange
var enumerator = Substitute.For <IDirectoryEnumerator>();
var expected = new[] { "1.cs", "2.cs" };
enumerator.Files("root", "*.cs", true).Returns(expected);
var sut = new FolderParser(enumerator);
// Act
var actual = sut.Files("root");
// Assert
Assert.Equal(expected, actual);
}
private void SourceFilesTree_DragDrop(object sender, DragEventArgs e)
{
var fileData = e.Data.GetData("FileDrop") as string[];
if (fileData != null)
{
var folderParser = new FolderParser();
foreach (var item in fileData)
{
if (Directory.Exists(item))
{
ImportSourceFiles(folderParser, item);
}
}
}
}
private void ImportFilesClipboardToolStripMenuItem_Click(object sender, EventArgs e)
{
var clipTxt = Clipboard.GetText();
List <string> clipListData = null;
if (!string.IsNullOrWhiteSpace(clipTxt))
{
clipListData = clipTxt.ToListStr();
}
else
{
// Get the DataObject.
IDataObject data_object = Clipboard.GetDataObject();
if (data_object != null)
{
var fileDropData = data_object.GetData(DataFormats.FileDrop, true) as string[];
if (fileDropData != null)
{
clipListData = new List <string>();
foreach (var item in fileDropData)
{
clipListData.Add(item);
}
}
}
}
if (clipListData != null && clipListData.Count > 0)
{
for (int index = 0; index < clipListData.Count; index++)
{
if (Directory.Exists(clipListData[index]))
{
var folderParser = new FolderParser();
ImportSourceFolder(folderParser, clipListData[index]);
}
}
}
else
{
ErrorLog.Inst.ShowError("Clipboad data not supported");
}
}
public void PaserWordFromDir(string path, string typeFile = "wav")
{
listPaths = new Dictionary <int, string>();
listWords = new Dictionary <int, string>();
int count = 0;
FolderParser dirParser = new FolderParser(path);
List <string> dirs = dirParser.Folders;
foreach (string dir in dirs)
{
ParserFiles fileParser = new ParserFiles(dir, typeFile);
List <string> files = fileParser.Names;
foreach (string file in files)
{
listWords.Add(count, Path.GetFileNameWithoutExtension(file));
listPaths.Add(count, file);
count++;
}
}
}
public void ParseMultipleFolderNode()
{
var mock = new Mock <FolderService>();
mock.Setup(s => s.Create(It.IsAny <string>(), It.IsAny <string>())).Verifiable();
var service = mock.Object;
var parser = new FolderParser(service);
var xmlDoc = new XmlDocument();
using (Stream stream = Assembly.GetExecutingAssembly()
.GetManifestResourceStream("RsPackage.Testing.Resources.MultiLevelSample.xml"))
using (StreamReader reader = new StreamReader(stream))
xmlDoc.Load(reader);
var root = xmlDoc.FirstChild.NextSibling;
parser.Execute(root);
Mock.Get(service).Verify(s => s.Create(It.IsAny <string>(), It.IsAny <string>()), Times.Exactly(4));
}
private void FileConvertLabel_DragDrop(object sender, DragEventArgs e)
{
var fileData = e.Data.GetData("FileDrop") as string[];
if (fileData != null)
{
var folderParser = new FolderParser();
foreach (var item in fileData)
{
if (Directory.Exists(item))
{
string selectedSrcFolder = item;
string selectedDestFolder = item + "_Fixed";
if (Directory.Exists(selectedDestFolder))
{
Directory.CreateDirectory(selectedDestFolder);
}
var progressBar = ProgressDialog.Run(this, "Fixing Directory and File Names ...", "Fixing Directory and File Names \"{0}\"", selectedDestFolder);
try
{
FileUtility.Inst.CopyAllFix(string.Format(@"\\?\{0}", selectedSrcFolder.Trim()), string.Format(@"\\?\{0}", selectedDestFolder.Trim()));
}
catch (Exception ex)
{
ErrorLog.Inst.ShowError("Failed to fix directory names : {0}", ex.Message);
}
Settings.Default.ImportLastPath = selectedDestFolder;
progressBar.ShutDown();
ErrorLog.Inst.ShowInfo("File name conversion completed. \"{0}\"", selectedDestFolder);
}
}
}
}
