public void TestRegexCaptureReplace(bool useLongPath)
{
string destFolder = useLongPath ? GetLongPathDestination(Guid.NewGuid().ToString()) : destinationFolder;
Utils.CopyFiles(Path.Combine(sourceFolder, "TestCase3"), Path.Combine(destinationFolder, "TestCase3"), null, null);
GrepCore core = new GrepCore();
List <GrepSearchResult> results = core.Search(Directory.GetFiles(Path.Combine(destFolder, "TestCase3"), "test-file-code.cs"),
SearchType.Regex, @"-(\d)", GrepSearchOption.None, -1);
Assert.Single(results);
Assert.Single(results[0].SearchResults.Where(r => !r.IsContext));
// mark all matches for replace
foreach (var match in results[0].Matches)
{
match.ReplaceMatch = true;
}
string testFile = Path.Combine(destFolder, @"TestCase3\test-file-code.cs");
List <ReplaceDef> files = new List <ReplaceDef>
{
new ReplaceDef(testFile, results[0].Matches)
};
core.Replace(files, SearchType.Regex, @"-(\d)", @"$1", GrepSearchOption.None, -1);
string content = File.ReadAllText(testFile, Encoding.ASCII);
Assert.DoesNotContain("= -1;", content);
Assert.Contains("= 1;", content);
core.Undo(files);
content = File.ReadAllText(testFile, Encoding.ASCII);
Assert.DoesNotContain("= 1;", content);
Assert.Contains("= -1;", content);
}
public static IGrepEngine GetReplaceEngine(string fileName, GrepEngineInitParams param, FileFilter filter)
{
Debug.Assert(param != null);
Debug.Assert(filter != null);
LoadPlugins();
string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.');
lock (lockObj)
{
if (fileTypeEngines.ContainsKey(fileExtension) && !fileTypeEngines[fileExtension].IsSearchOnly)
{
IGrepEngine engine = fileTypeEngines[fileExtension].CreateEngine();
if (engine != null && engine.Initialize(param, filter))
{
loadedEngines.Add(engine);
return(engine);
}
else
{
failedEngines[engine.GetType().Name] = "Failed to initialize the plugin. See error log for details.";
return(GetPlainTextEngine(fileExtension, param, filter));
}
}
else
{
return(GetPlainTextEngine(fileExtension, param, filter));
}
}
}
public static void Export(WebLibraryDetail web_library_detail, List <PDFDocument> pdf_documents)
{
FeatureTrackingManager.Instance.UseFeature(Features.Library_Export);
// Get the directory
string initial_directory = null;
if (null == initial_directory)
{
initial_directory = Path.GetDirectoryName(ConfigurationManager.Instance.ConfigurationRecord.System_LastLibraryExportFolder);
}
if (null == initial_directory)
{
initial_directory = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
}
using (CommonOpenFileDialog dialog = new CommonOpenFileDialog())
{
dialog.IsFolderPicker = true;
dialog.Title = "Please select the folder to which you wish to export your entire Qiqqa library.";
dialog.DefaultDirectory = initial_directory;
CommonFileDialogResult result = dialog.ShowDialog();
if (result == CommonFileDialogResult.Ok)
{
// Remember the filename for next time
string base_path = dialog.FileName;
ConfigurationManager.Instance.ConfigurationRecord.System_LastLibraryExportFolder = base_path;
ConfigurationManager.Instance.ConfigurationRecord_Bindable.NotifyPropertyChanged(nameof(ConfigurationManager.Instance.ConfigurationRecord.System_LastLibraryExportFolder));
SafeThreadPool.QueueUserWorkItem(o => Export(web_library_detail, pdf_documents, base_path));
}
}
}
private static void AddPathToSevenZip(Dictionary <string, System.IO.Stream> files, IList <System.IO.Stream> streams, string snapshotPath, int volumePathLength, string vmPath)
{
var srcPath = Path.Combine(snapshotPath, vmPath.Substring(volumePathLength));
if (Directory.Exists(srcPath))
{
foreach (var srcChildPath in Directory.GetFileSystemEntries(srcPath))
{
var srcChildPathRel = srcChildPath.Substring(snapshotPath.EndsWith(Path.PathSeparator.ToString(), StringComparison.CurrentCultureIgnoreCase) ? snapshotPath.Length : snapshotPath.Length + 1);
var childPath = Path.Combine(vmPath.Substring(0, volumePathLength), srcChildPathRel);
AddPathToSevenZip(files, streams, snapshotPath, volumePathLength, childPath);
}
}
else if (File.Exists(srcPath))
{
var s = File.OpenRead(srcPath);
files.Add(vmPath.Substring(volumePathLength), s);
streams.Add(s);
}
else
{
var lowerPath = srcPath.ToLowerInvariant();
var isIgnorable = lowerPath.EndsWith(".avhdx") || lowerPath.EndsWith(".vmrs") ||
lowerPath.EndsWith(".bin") || lowerPath.EndsWith(".vsv");
if (!isIgnorable)
{
throw new Exception($"Entry \"{srcPath}\" not found in snapshot");
}
}
}
public async Task DownloadMissingArchives(List <Archive> missing, bool download = true)
{
if (download)
{
var result = SendDownloadMetrics(missing);
foreach (var a in missing.Where(a => a.State.GetType() == typeof(ManualDownloader.State)))
{
var outputPath = DownloadFolder.Combine(a.Name);
await a.State.Download(a, outputPath);
}
}
await missing.Where(a => a.State.GetType() != typeof(ManualDownloader.State))
.PMap(Queue, UpdateTracker, async archive =>
{
Info($"Downloading {archive.Name}");
var outputPath = DownloadFolder.Combine(archive.Name);
if (download)
{
if (outputPath.Exists)
{
var origName = Path.GetFileNameWithoutExtension(archive.Name);
var ext = Path.GetExtension(archive.Name);
var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex();
outputPath = DownloadFolder.Combine(origName + "_" + uniqueKey + "_" + ext);
await outputPath.DeleteAsync();
}
}
return(await DownloadArchive(archive, download, outputPath));
});
}
private void Validate()
{
if (Path.IsPathRooted(_path))
{
throw new InvalidDataException($"Cannot create relative path from absolute path string, got {_path}");
}
}
public static async Task <BethesdaNetData> Login()
{
var game = Path.Combine(Game.SkyrimSpecialEdition.MetaData().GameLocation(), "SkyrimSE.exe");
var info = new ProcessStartInfo
{
FileName = @"Downloaders\BethesdaNet\bethnetlogin.exe",
Arguments = $"\"{game}\" SkyrimSE.exe",
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
var process = Process.Start(info);
ChildProcessTracker.AddProcess(process);
string last_line = "";
while (true)
{
var line = await process.StandardOutput.ReadLineAsync();
if (line == null)
{
break;
}
last_line = line;
}
var result = last_line.FromJSONString <BethesdaNetData>();
result.ToEcryptedJson(DataName);
return(result);
}
private void GenericCustomiseChooser(string title, string filename)
{
OpenFileDialog dialog = new OpenFileDialog();
dialog.Filter = "Image files|*.jpeg;*.jpg;*.png;*.gif;*.bmp" + "|" + "All files|*.*";
dialog.CheckFileExists = true;
dialog.Multiselect = false;
dialog.Title = title;
//dialog.FileName = filename;
if (true == dialog.ShowDialog())
{
// Copy the new file into place, if it is another file than the one we already have:
filename = Path.GetFullPath(filename);
string new_filename = Path.GetFullPath(dialog.FileName);
if (0 != new_filename.CompareTo(filename))
{
File.Delete(filename);
File.Copy(new_filename, filename);
}
}
else
{
File.Delete(filename);
}
UpdateLibraryStatistics();
}
public override async ValueTask <Directive> Run(RawSourceFile source)
{
var filename = Path.GetFileName(source.Path);
var gameFile = Path.Combine(_mo2Compiler.GamePath, "Data", filename);
if (!Consts.GameESMs.Contains(filename) || !source.Path.StartsWith("mods\\") ||
!File.Exists(gameFile))
{
return(null);
}
Utils.Log(
$"An ESM named {filename} was found in a mod that shares a name with one of the core game ESMs, it is assumed this is a cleaned ESM and it will be binary patched");
var result = source.EvolveTo <CleanedESM>();
result.SourceESMHash = _compiler.VFS.Index.ByRootPath[gameFile].Hash;
Utils.Status($"Generating patch of {filename}");
using (var ms = new MemoryStream())
{
Utils.CreatePatch(File.ReadAllBytes(gameFile), File.ReadAllBytes(source.AbsolutePath), ms);
var data = ms.ToArray();
result.SourceDataID = _compiler.IncludeFile(data);
Utils.Log($"Generated a {data.Length} byte patch for {filename}");
}
return(result);
}
public IntranetLibraryDB(string base_path)
{
this.base_path = base_path;
library_path = IntranetLibraryTools.GetLibraryMetadataPath(base_path);
// Copy a library into place...
if (!File.Exists(library_path))
{
Logging.Warn("Intranet Library metadata db does not exist so copying the template to {0}", library_path);
string library_metadata_template_path = Path.GetFullPath(Path.Combine(ConfigurationManager.Instance.StartupDirectoryForQiqqa, @"DocumentLibrary/IntranetLibraryStuff/IntranetLibrary.Metadata.Template.s3db"));
if (!File.Exists(library_metadata_template_path))
{
throw new Exception($"Sync template file '{library_metadata_template_path}' does not exist!");
}
string basedir = Path.GetDirectoryName(library_path);
if (!Directory.Exists(basedir))
{
throw new Exception($"Sync target directory '{basedir}' for Qiqqa database '{library_path}' does not exist!");
}
try
{
File.Copy(library_metadata_template_path, library_path);
}
catch (Exception ex)
{
Logging.Error(ex, "Error 0x{2:08X}: Failed to write the sync template '{0}' to sync target directory '{1}'", library_metadata_template_path, basedir, ex.HResult);
throw;
}
}
}
public async Task SetScreenSizeTest()
{
var profile = utils.AddProfile();
var mod = utils.AddMod("dummy");
utils.Configure();
File.WriteAllLines(Path.Combine(utils.MO2Folder, "profiles", profile, "somegameprefs.ini"),
new List <string>
{
// Beth inis are messy, let's make ours just as messy to catch some parse failures
"[Display]",
"foo=4",
"[Display]",
"STestFile=f",
"STestFile=",
"iSize H=3",
"iSize W=-200",
"[Display]",
"foo=4"
});
var modlist = await CompileAndInstall(profile);
var ini = Path.Combine(utils.InstallFolder, "profiles", profile, "somegameprefs.ini").LoadIniFile();
Assert.AreEqual(System.Windows.SystemParameters.PrimaryScreenHeight.ToString(), ini?.Display?["iSize H"]);
Assert.AreEqual(System.Windows.SystemParameters.PrimaryScreenWidth.ToString(), ini?.Display?["iSize W"]);
}
internal static bool IsSupportedImagePath(string filename)
{
string extension = Path.GetExtension(filename.ToLower(CultureInfo.CurrentCulture));
if (0 == extension.CompareTo(".jpg"))
{
return(true);
}
if (0 == extension.CompareTo(".png"))
{
return(true);
}
if (0 == extension.CompareTo(".jpeg"))
{
return(true);
}
if (0 == extension.CompareTo(".gif"))
{
return(true);
}
if (0 == extension.CompareTo(".bmp"))
{
return(true);
}
return(false);
}
public CSLProcessorOutputConsumer(string script_directory, string citations_javascript, BibliographyReadyDelegate brd, object user_argument)
{
this.citations_javascript = citations_javascript;
this.brd = brd;
this.user_argument = user_argument;
// Create the browser
Logging.Info("Creating web browser for InCite CSL processing");
web_browser = new GeckoWebBrowser();
web_browser.CreateControl();
// Add the name of the script to run
script_directory = Path.GetFullPath(Path.Combine(script_directory, @"runengine.html"));
script_directory = script_directory.Replace(@"\\", @"\");
script_directory = script_directory.Replace(@"//", @"/");
Uri uri = new Uri(script_directory);
Logging.Info("CSLProcessorOutputConsumer is about to browse to {0}", uri);
// This is the only way we can communicate from JavaScript to .NET!!
web_browser.EnableConsoleMessageNotfication();
web_browser.ConsoleMessage += web_browser_ConsoleMessage;
// Kick off citeproc computation
web_browser.Navigate(uri.ToString());
}
public string AddManualDownload(Dictionary <string, byte[]> contents)
{
var name = RandomName() + ".zip";
using (FileStream fs = new FileStream(Path.Combine(DownloadsFolder, name), FileMode.Create))
using (ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Create))
{
contents.Do(kv =>
{
var entry = archive.CreateEntry(kv.Key);
using (var os = entry.Open())
os.Write(kv.Value, 0, kv.Value.Length);
});
}
File.WriteAllLines(Path.Combine(DownloadsFolder, name + ".meta"),
new string[]
{
"[General]",
"manualURL=<TESTING>"
});
return(name);
}
public void TestSearchAndReplaceXPathWithMissingXmlDeclaration(bool useLongPath)
{
string destFolder = useLongPath ? GetLongPathDestination(Guid.NewGuid().ToString()) : destinationFolder;
Utils.CopyFiles(Path.Combine(sourceFolder, "TestCase4"), Path.Combine(destFolder, "TestCase4"), null, null);
GrepCore core = new GrepCore();
List <GrepSearchResult> results = core.Search(Directory.GetFiles(Path.Combine(destFolder, "TestCase4"), "books_no_decl.xml"),
SearchType.XPath, "(//@category)[2]", GrepSearchOption.None, -1);
Assert.Single(results);
Assert.Single(results[0].Matches);
// mark all matches for replace
foreach (var match in results[0].Matches)
{
match.ReplaceMatch = true;
}
string testFile = Path.Combine(destinationFolder, "TestCase4", "books_no_decl.xml");
List <ReplaceDef> files = new List <ReplaceDef>
{
new ReplaceDef(testFile, results[0].Matches)
};
core.Replace(files, SearchType.XPath, "(//@category)[2]", "general", GrepSearchOption.None, -1);
var fileContent = File.ReadAllLines(testFile, Encoding.UTF8);
Assert.Equal(37, fileContent.Length);
Assert.Equal("<bookstore>", fileContent[0]);
Assert.Equal(" <book category=\"general\">", fileContent[7]);
}
/// <summary>
/// Saves settings to location specified
/// </summary>
/// <param name="path">Path to settings file</param>
public void Save(string path)
{
try
{
if (!Directory.Exists(Path.GetDirectoryName(path)))
{
Directory.CreateDirectory(Path.GetDirectoryName(path));
}
lock (this)
{
// Create temp file in case save crashes
using (FileStream stream = File.OpenWrite(path + "~"))
using (XmlWriter xmlStream = XmlWriter.Create(stream, new XmlWriterSettings {
Indent = true
}))
{
if (xmlStream == null)
{
return;
}
XmlSerializer serializer = new XmlSerializer(typeof(SerializableDictionary));
serializer.Serialize(xmlStream, this);
}
File.Copy(path + "~", path, true);
Utils.DeleteFile(path + "~");
}
}
catch (Exception ex)
{
logger.Log <Exception>(LogLevel.Error, "Failed to load settings", ex);
}
}
public string GetLongPathDestination(string leafFolder)
{
var parts = new string[]
{
destinationFolder,
new string('a', 50),
new string('b', 50),
new string('c', 50),
new string('d', 50),
new string('e', 50),
new string('f', 50),
new string('g', 50),
new string('h', 50),
leafFolder
};
destinationFolder = Path.Combine(parts);
if (!Directory.Exists(destinationFolder))
{
Directory.CreateDirectory(destinationFolder);
}
return(destinationFolder);
}
public static string NormalizePath(this string path)
{
//https://stackoverflow.com/questions/2281531/how-can-i-compare-directory-paths-in-c
return(Path.GetFullPath(new Uri(path).LocalPath)
.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
.ToUpperInvariant());
}
public void TestSearchAndFilteredReplaceXPathWithMissingXmlDeclaration(bool useLongPath)
{
string destFolder = useLongPath ? GetLongPathDestination(Guid.NewGuid().ToString()) : destinationFolder;
Utils.CopyFiles(Path.Combine(sourceFolder, "TestCase4"), Path.Combine(destFolder, "TestCase4"), null, null);
GrepCore core = new GrepCore();
List <GrepSearchResult> results = core.Search(Directory.GetFiles(Path.Combine(destFolder, "TestCase4"), "books_no_decl.xml"),
SearchType.XPath, "(//@currency)", GrepSearchOption.None, -1);
Assert.Single(results);
Assert.Equal(5, results[0].Matches.Count);
// mark 2nd and 4th matches for replace
results[0].Matches[1].ReplaceMatch = true;
results[0].Matches[3].ReplaceMatch = true;
string testFile = Path.Combine(destinationFolder, "TestCase4", "books_no_decl.xml");
List <ReplaceDef> files = new List <ReplaceDef>
{
new ReplaceDef(testFile, results[0].Matches)
};
core.Replace(files, SearchType.XPath, "(//@currency)", "EUR", GrepSearchOption.None, -1);
var fileContent = File.ReadAllText(testFile, Encoding.UTF8);
Assert.Equal(2, Regex.Matches(fileContent, "EUR").Count);
}
private static string ConvertNormalFilenameToZoteroFilename(string bibtex_base_filename, string filename)
{
string bibtex_base_path = Path.GetDirectoryName(bibtex_base_filename);
string file = Path.GetFileName(filename);
string path = filename;
/// Try shrink it to a relative path
if (path.StartsWith(bibtex_base_path))
{
path = path.Substring(bibtex_base_path.Length + 1);
}
else
{
Logging.Warn("Unable to convert path to Zotero form: {0}", path);
}
// Replace the slashes
path = path.Replace('\\', '/');
// Concatenate
string export_string =
""
+ file
+ ":"
+ path
+ ":"
+ "application/pdf";
return(export_string);
}
public void TestReplaceWithNewLineWorks(bool useLongPath)
{
string destFolder = useLongPath ? GetLongPathDestination(Guid.NewGuid().ToString()) : destinationFolder;
Utils.CopyFiles(Path.Combine(sourceFolder, "TestCase8"), Path.Combine(destFolder, "TestCase8"), null, null);
GrepCore core = new GrepCore();
List <GrepSearchResult> results = core.Search(Directory.GetFiles(Path.Combine(destFolder, "TestCase8"), "test.txt"), SearchType.Regex, "here", GrepSearchOption.None, -1);
Assert.Single(results);
Assert.Single(results[0].SearchResults);
// mark all matches for replace
foreach (var match in results[0].Matches)
{
match.ReplaceMatch = true;
}
string testFile = Path.Combine(destFolder, @"TestCase8\test.txt");
List <ReplaceDef> files = new List <ReplaceDef>
{
new ReplaceDef(testFile, results[0].Matches)
};
core.Replace(files, SearchType.Regex, "here", "\\\\n", GrepSearchOption.None, -1);
Assert.Equal(2, File.ReadAllText(testFile, Encoding.ASCII).Trim().Split('\n').Length);
}
public static async Task UploadPackagedInis(WorkQueue queue, IEnumerable <Archive> archives)
{
archives = archives.ToArray(); // defensive copy
Utils.Log($"Packaging {archives.Count()} inis");
try
{
await using var ms = new MemoryStream();
using (var z = new ZipArchive(ms, ZipArchiveMode.Create, true))
{
foreach (var e in archives)
{
if (e.State == null)
{
continue;
}
var entry = z.CreateEntry(Path.GetFileName(e.Name));
await using var os = entry.Open();
await os.WriteAsync(Encoding.UTF8.GetBytes(string.Join("\n", e.State.GetMetaIni())));
}
}
var webClient = new WebClient();
await webClient.UploadDataTaskAsync($"https://{Consts.WabbajackCacheHostname}/indexed_files/notify",
"POST", ms.ToArray());
}
catch (Exception ex)
{
Utils.Log(ex.ToString());
}
}
public void TestSearchAndFilteredReplace(SearchType type, GrepSearchOption option, string searchFor, string replaceWith)
{
Utils.CopyFiles(Path.Combine(sourceFolder, "TestCase15"), Path.Combine(destinationFolder, "TestCase15"), null, null);
GrepCore core = new GrepCore();
List <GrepSearchResult> results = core.Search(Directory.GetFiles(Path.Combine(destinationFolder, "TestCase15"), "books.xml"),
type, searchFor, option, -1);
Assert.Single(results);
Assert.Equal(2, results[0].Matches.Count);
// mark only the second match for replace
results[0].Matches[1].ReplaceMatch = true;
string testFile = Path.Combine(destinationFolder, "TestCase15", "books.xml");
List <ReplaceDef> files = new List <ReplaceDef>
{
new ReplaceDef(testFile, results[0].Matches)
};
core.Replace(files, type, searchFor, replaceWith, option, -1);
var fileContent = File.ReadAllText(testFile, Encoding.UTF8);
Assert.Contains("<year>2003</year>", fileContent);
Assert.Single(Regex.Matches(fileContent, "2002"));
Assert.Single(Regex.Matches(fileContent, "2003"));
}
public List <GrepSearchResult> Search(string file, string searchPattern, SearchType searchType, GrepSearchOption searchOptions, Encoding encoding)
{
string tempFolder = Path.Combine(Utils.GetTempFolder(), "dnGREP-PDF");
try
{
// Extract text
string tempFile = ExtractText(file);
if (!File.Exists(tempFile))
{
throw new ApplicationException("pdftotext failed to create text file.");
}
// GrepCore cannot check encoding of the original pdf file. If the encoding parameter is not default
// then it is the user-specified code page. If the encoding parameter *is* the default,
// then it most likely not been set, so get the encoding of the extracted text file:
if (encoding == Encoding.Default)
{
encoding = Utils.GetFileEncoding(tempFile);
}
IGrepEngine engine = GrepEngineFactory.GetSearchEngine(tempFile, initParams, FileFilter, searchType);
List <GrepSearchResult> results = engine.Search(tempFile, searchPattern, searchType, searchOptions, encoding);
if (results.Count > 0)
{
using (FileStream reader = File.Open(tempFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (StreamReader streamReader = new StreamReader(reader, encoding))
{
foreach (var result in results)
{
result.SearchResults = Utils.GetLinesEx(streamReader, result.Matches, initParams.LinesBefore, initParams.LinesAfter);
}
}
foreach (GrepSearchResult result in results)
{
result.ReadOnly = true;
if (file.Contains(tempFolder))
{
result.FileNameDisplayed = file.Substring(tempFolder.Length + 1);
}
else
{
result.FileNameDisplayed = file;
}
result.FileNameReal = file;
}
}
GrepEngineFactory.ReturnToPool(tempFile, engine);
return(results);
}
catch (Exception ex)
{
logger.Error(ex, $"Failed to search inside PDF file: {ex.Message}");
return(new List <GrepSearchResult>());
}
}
internal static void BuildReport(Library library, List <PDFDocument> pdf_documents)
{
AnnotationReportOptions annotation_report_options = new AnnotationReportOptions();
List <AnnotationWorkGenerator.AnnotationWork> annotation_works = AnnotationWorkGenerator.GenerateAnnotationWorks(library, pdf_documents, annotation_report_options);
IEnumerable <AnnotationJSON> annotation_jsons = annotation_works.Select(annotation_work =>
new AnnotationJSON
{
fingerprint = annotation_work.pdf_document.Fingerprint,
title = annotation_work.pdf_document.TitleCombined,
page = annotation_work.pdf_annotation.Page,
left = annotation_work.pdf_annotation.Left,
top = annotation_work.pdf_annotation.Top,
width = annotation_work.pdf_annotation.Width,
height = annotation_work.pdf_annotation.Height,
tags = annotation_work.pdf_annotation.Tags,
text = annotation_work.pdf_annotation.Text,
}
);
string json = JsonConvert.SerializeObject(annotation_jsons, Formatting.Indented);
string filename = Path.GetTempFileName() + ".json.txt";
File.WriteAllText(filename, json);
Process.Start(filename);
}
public Action Stage(IEnumerable <VirtualFile> files)
{
var grouped = files.SelectMany(f => f.FilesInPath)
.Distinct()
.Where(f => f.ParentArchive != null)
.GroupBy(f => f.ParentArchive)
.OrderBy(f => f.Key == null ? 0 : f.Key.Paths.Length)
.ToList();
var Paths = new List <string>();
foreach (var group in grouped)
{
var tmp_path = Path.Combine(_stagedRoot, Guid.NewGuid().ToString());
FileExtractor.ExtractAll(group.Key.StagedPath, tmp_path);
Paths.Add(tmp_path);
foreach (var file in group)
{
file._stagedPath = Path.Combine(tmp_path, file.Paths[group.Key.Paths.Length]);
}
}
return(() =>
{
Paths.Do(p =>
{
if (Directory.Exists(p))
{
DeleteDirectory(p);
}
});
});
}
public string ScrapeURLToFile(string url, bool force_download = false, Dictionary <string, string> additional_headers = null)
{
string cache_key = StreamMD5.FromText(url);
string directory = Path.GetFullPath(Path.Combine(base_directory, cache_key.Substring(0, 2)));
string filename = Path.GetFullPath(Path.Combine(directory, cache_key));
if (!File.Exists(filename) || force_download)
{
Utilities.Files.DirectoryTools.CreateDirectory(directory);
// Crude throttle
if (true)
{
while (DateTime.UtcNow.Subtract(last_scrape_time).TotalMilliseconds < throttle_ms)
{
Thread.Sleep(50);
}
last_scrape_time = DateTime.UtcNow;
}
Logging.Info("Downloading from {0}", url);
using (WebClient client = new WebClient())
{
if (null != additional_headers)
{
foreach (var pair in additional_headers)
{
client.Headers.Add(pair.Key, pair.Value);
}
}
if (!String.IsNullOrEmpty(userAgent))
{
client.Headers.Add("User-agent", userAgent);
}
string temp_filename = filename + ".tmp";
try
{
client.DownloadFile(url, temp_filename);
}
catch (WebException ex)
{
File.WriteAllText(temp_filename, ex.ToString());
}
File.Delete(filename);
File.Move(temp_filename, filename);
}
string filename_manifest = Path.GetFullPath(Path.Combine(base_directory, @"manifest.txt"));
string manifest_line = String.Format("{0}\t{1}", cache_key, url);
using (StreamWriter sw = File.AppendText(filename_manifest))
{
sw.WriteLine(manifest_line);
}
}
return(filename);
}
private void ChooseCSLFile(CSLFileSource csl_file_source)
{
OpenFileDialog dialog = new OpenFileDialog();
dialog.Filter = "CSL style files|*.csl" + "|" + "All files|*.*";
dialog.CheckFileExists = true;
dialog.Multiselect = false;
string filename = null;
string directory = null;
// Decide what we want to use as the CSL file location
if (CSLFileSource.TEXTBOX == csl_file_source)
{
try
{
CorrectStyleFilenameForNewDirectoryLocation();
directory = Path.GetDirectoryName(ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile);
filename = Path.GetFileName(ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile);
// If the directory no longer exists, kill our memory of it. We will pick the default directory again.
if (!Directory.Exists(directory))
{
directory = null;
}
}
catch { }
}
if (null == directory)
{
directory = PDFDocumentCitingTools.BASE_STYLE_DIRECTORY;
}
// Set the dialog defaults
dialog.FileName = filename;
dialog.InitialDirectory = directory;
// Get the user response
if (true == dialog.ShowDialog())
{
ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile = dialog.FileName;
ConfigurationManager.Instance.ConfigurationRecord_Bindable.NotifyPropertyChanged(() => ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile);
// Check if it is a dependent style
string style_xml_filename = ConfigurationManager.Instance.ConfigurationRecord.InCite_LastStyleFile;
string root_style_filename = DependentStyleDetector.GetRootStyleFilename(style_xml_filename);
if (root_style_filename != style_xml_filename)
{
if (null != root_style_filename)
{
MessageBoxes.Info("This style is dependent on another style, so InCite will be using this file instead:\n" + root_style_filename);
}
else
{
MessageBoxes.Info("This style is dependent on another style that is not available on your computer. Please download it before proceeding.");
}
}
}
}
private string ExtractText(string pdfFilePath)
{
string tempFolder = Path.Combine(Utils.GetTempFolder(), "dnGREP-PDF");
if (!Directory.Exists(tempFolder))
{
Directory.CreateDirectory(tempFolder);
}
string tempFileName = Path.Combine(tempFolder, Path.GetFileNameWithoutExtension(pdfFilePath) + ".txt");
if (pdfFilePath.Length > 260 && !pdfFilePath.StartsWith(@"\\?\"))
{
pdfFilePath = @"\\?\" + pdfFilePath;
}
string options = GrepSettings.Instance.Get <string>(GrepSettings.Key.PdfToTextOptions);
using (Process process = new Process())
{
// use command prompt
process.StartInfo.FileName = pathToPdfToText;
process.StartInfo.Arguments = string.Format("{0} \"{1}\" \"{2}\"", options, pdfFilePath, tempFileName);
process.StartInfo.UseShellExecute = false;
process.StartInfo.WorkingDirectory = Utils.GetCurrentPath(typeof(GrepEnginePdf));
process.StartInfo.CreateNoWindow = true;
// start cmd prompt, execute command
process.Start();
process.WaitForExit();
if (process.ExitCode == 0)
{
return(tempFileName);
}
else
{
string errorMessage = string.Empty;
switch (process.ExitCode)
{
case 1:
errorMessage = "Error opening PDF file";
break;
case 2:
errorMessage = "Error opening an output file";
break;
case 3:
errorMessage = "Error related to PDF permissions";
break;
default:
errorMessage = "Unknown error";
break;
}
throw new Exception($"pdftotext returned '{errorMessage}' converting '{pdfFilePath}'");
}
}
}
public void WriteMasterList()
{
Logging.Info("+WriteMasterList");
string filename_temp = Path.GetTempFileName();
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
FlushAllWords_LOCK();
PurgeAllWords_LOCK();
using (FileStream fs = File.Open(filename_temp, FileMode.Create))
{
Headers headers = new Headers();
// First the documents
{
headers.documents = new List <DocumentMapHeader>();
foreach (var pair in fingerprint_to_document_ids)
{
DocumentMapHeader header = new DocumentMapHeader
{
Fingerprint = pair.Key,
DocumentId = pair.Value
};
headers.documents.Add(header);
}
}
// Then the words
{
headers.words = new List <WordMapHeader>();
foreach (WordInWordIndex word_in_word_index in word_in_word_indexes)
{
WordMapHeader header = new WordMapHeader
{
Word = word_in_word_index.Word,
WordId = word_in_word_index.WordId,
DocCount = word_in_word_index.DocumentCount
};
headers.words.Add(header);
}
}
Serializer.Serialize <Headers>(fs, headers);
}
}
Logging.Info("-WriteMasterList");
// Move the temp file over the library filename
Directory.CreateDirectory(Path.GetDirectoryName(GetFilename_MasterList()));
FileTools.MoveSafelyWithOverwriting(filename_temp, GetFilename_MasterList());
// Write the version of the index
File.WriteAllText(VersionFilename, INDEX_VERSION);
}
