private void CreateZipFile(string finalZipFolder, string finalZipFile, string tempFolder, bool createWithPassword, string id)
{
try
{
if (Directory.Exists(finalZipFolder))
{
Directory.Delete(finalZipFolder, true);
}
if (File.Exists(finalZipFile))
{
File.Delete(finalZipFile);
}
Directory.GetParent(finalZipFolder)?.Create();
Directory.Move(tempFolder, finalZipFolder);
if (createWithPassword)
{
CreateZipFileWithPasswordFromDirectory(finalZipFolder, finalZipFile, passwordHelper.GetHashPassword(id));
}
else
{
ZipFile.CreateFromDirectory(finalZipFolder, finalZipFile);
}
}
finally
{
if (Directory.Exists(finalZipFolder))
{
Directory.Delete(finalZipFolder, true);
}
}
}
public static string GetGuildLogZipped(ulong guildId)
{
if (!Directory.Exists($@"{Environment.CurrentDirectory}/ZipWorkspace"))
{
Directory.CreateDirectory($@"{Environment.CurrentDirectory}/ZipWorkspace");
}
else
{
Directory.Delete($@"{Environment.CurrentDirectory}/ZipWorkspace", true);
Directory.CreateDirectory($@"{Environment.CurrentDirectory}/ZipWorkspace");
}
try
{
if (File.Exists($@"{Environment.CurrentDirectory}/guild_{guildId}_log.zip"))
{
File.Delete($@"{Environment.CurrentDirectory}/guild_{guildId}_log.zip");
}
File.Copy($@"{Environment.CurrentDirectory}/Logs/guild_{guildId}.log", $@"{ Environment.CurrentDirectory}/ZipWorkspace/guild_{guildId}.log");
ZipFile.CreateFromDirectory($@"{Environment.CurrentDirectory}/ZipWorkspace", $@"{Environment.CurrentDirectory}/guild_{guildId}_log.zip");
return($@"{Environment.CurrentDirectory}/guild_{guildId}_log.zip");
}
catch (Exception e)
{
return(e.ToString());
}
}
public void Process(string url, string foldername)
{
var zip = Download(url, foldername);
RemoveExistingDocumentation(RootFolder);
ZipFile.ExtractToDirectory(zip, RootFolder);
var unzippedPath = RootFolder + "\\UmbracoDocs-master\\";
foreach (var directory in new DirectoryInfo(unzippedPath).GetDirectories())
{
Directory.Move(directory.FullName, RootFolder + "\\" + directory.Name);
}
foreach (var file in new DirectoryInfo(unzippedPath).GetFiles())
{
File.Move(file.FullName, RootFolder + "\\" + file.Name);
}
Directory.Delete(unzippedPath, true);
BuildSitemap(foldername);
//YUCK, this is horrible but unfortunately the way that the doc indexes are setup are not with
// a consistent integer id per document. I'm sure we can make that happen but I don't have time right now.
ExamineManager.Instance.IndexProviderCollection["documentationIndexer"].RebuildIndex();
}
public void AssembleDataTableFromFileArchive()
{
var zip = Path.Combine(TestContext.CurrentContext.TestDirectory, "TestData.zip");
var dir = Path.Combine(TestContext.CurrentContext.TestDirectory, "TestData");
if (File.Exists(zip))
{
File.Delete(zip);
}
ZipFile.CreateFromDirectory(dir, zip);
var fileCount = Directory.GetFiles(dir, "*.dcm").Count();
var source = new DicomFileCollectionSource();
source.FilenameField = "RelativeFileArchiveURI";
source.PreInitialize(new FlatFileToLoadDicomFileWorklist(new FlatFileToLoad(new FileInfo(zip))), new ThrowImmediatelyDataLoadEventListener());
var toMemory = new ToMemoryDataLoadEventListener(true);
var result = source.GetChunk(toMemory, new GracefulCancellationToken());
//processed every file once
Assert.AreEqual(fileCount, toMemory.LastProgressRecieivedByTaskName.Single().Value.Progress.Value);
Assert.Greater(result.Columns.Count, 0);
}
/// <summary>
/// Extracts the fulltext and adds the resulting text to the ArchiveRecord.
/// </summary>
/// <param name="mutationId">The mutation identifier.</param>
/// <param name="archiveRecord">The archive record.</param>
/// <returns><c>true</c> if successful, <c>false</c> otherwise.</returns>
public async Task <bool> ExtractFulltext(long mutationId, ArchiveRecord archiveRecord, int primaerdatenAuftragStatusId)
{
var packages = archiveRecord.PrimaryData;
var processingTimeForMissingFiles = 0L;
foreach (var repositoryPackage in packages.Where(p => !string.IsNullOrEmpty(p.PackageFileName)))
{
var packageFileName = Path.Combine(Settings.Default.PickupPath, repositoryPackage.PackageFileName);
var fi = new FileInfo(packageFileName);
var watch = Stopwatch.StartNew();
if (File.Exists(fi.FullName))
{
Log.Information("Found zip file {Name}. Starting to extract...", fi.Name);
var tempFolder = Path.Combine(fi.DirectoryName, fi.Name.Remove(fi.Name.Length - fi.Extension.Length));
try
{
ZipFile.ExtractToDirectory(packageFileName, tempFolder);
var sizeInBytesOnDisk = Directory.GetFiles(tempFolder, "*.*", SearchOption.AllDirectories).Select(f => new FileInfo(f).Length)
.Sum();
var status = new UpdatePrimaerdatenAuftragStatus
{
PrimaerdatenAuftragId = primaerdatenAuftragStatusId,
Service = AufbereitungsServices.AssetService,
Status = AufbereitungsStatusEnum.ZipEntpackt
};
await UpdatePrimaerdatenAuftragStatus(status);
await ProcessFiles(repositoryPackage.Files, Path.Combine(tempFolder, "content"), archiveRecord.ArchiveRecordId);
await ProcessFolders(repositoryPackage.Folders, Path.Combine(tempFolder, "content"), archiveRecord.ArchiveRecordId);
// if we are here everything is okay
Log.Information("Successfully processed (fulltext extracted) zip file {Name}", fi.Name);
processingTimeForMissingFiles += GetProcessingTimeOfIgnoredFilesInTicks(repositoryPackage.SizeInBytes - sizeInBytesOnDisk);
}
catch (Exception ex)
{
Log.Error(ex, "Unexpected error while extracting full text. Error Message is: {Message}", ex.Message);
return(false);
}
finally
{
// Delete the temp files
Directory.Delete(tempFolder, true);
File.Delete(packageFileName);
}
}
else
{
Log.Warning("Unable to find the zip file {packageFileName}. No text was extracted.", packageFileName);
return(false);
}
repositoryPackage.FulltextExtractionDuration = watch.ElapsedTicks + processingTimeForMissingFiles;
}
return(true);
}
public static string GetAllLogsZipped()
{
if (File.Exists($@"{Environment.CurrentDirectory}/Logs.zip"))
{
File.Delete($@"{Environment.CurrentDirectory}/Logs.zip");
}
ZipFile.CreateFromDirectory($@"{Environment.CurrentDirectory}/Logs", $@"{Environment.CurrentDirectory}/Logs.zip");
return($@"{Environment.CurrentDirectory}/Logs.zip");
}
public void Process(string url, string foldername)
{
var zip = Download(url, foldername);
var unzippedPath = string.Empty;
var branchAllowed = false;
using (var zipFile = ZipFile.OpenRead(zip))
{
var directory = zipFile.Entries.FirstOrDefault();
if (directory != null)
{
var branch = directory.FullName.Replace("/", string.Empty).Replace("UmbracoDocs-", string.Empty);
branchAllowed = IsBranchWhiteListed(branch);
if (branchAllowed)
{
unzippedPath = RootFolder + Path.DirectorySeparatorChar +
directory.FullName.Replace('/', Path.DirectorySeparatorChar);
if (Directory.Exists(unzippedPath))
{
//delete the directory we're trying to unzip to
RemoveUnzippedFiles(unzippedPath);
}
}
}
}
if (branchAllowed == false)
{
return;
}
ZipFile.ExtractToDirectory(zip, RootFolder);
RemoveUnzippedFiles(RootFolder, ignoreDirectories: new List <string> {
unzippedPath
});
foreach (var directory in new DirectoryInfo(unzippedPath).GetDirectories())
{
Directory.Move(directory.FullName, RootFolder + "\\" + directory.Name);
}
foreach (var file in new DirectoryInfo(unzippedPath).GetFiles())
{
File.Move(file.FullName, RootFolder + "\\" + file.Name);
}
Directory.Delete(unzippedPath, true);
BuildSitemap(foldername);
//YUCK, this is horrible but unfortunately the way that the doc indexes are setup are not with
// a consistent integer id per document. I'm sure we can make that happen but I don't have time right now.
ExamineManager.Instance.IndexProviderCollection["documentationIndexer"].RebuildIndex();
}
public static void SmartExtractFolder(string archive, string dest, bool overwrite = true)
{
var ext = Path.GetExtension(archive).ToLowerInvariant();
if (ext == ".tar.gz" || ext == ".tgz")
{
ExtractTGZ(archive, dest);
}
else
{
ZipFile.ExtractToDirectory(archive, dest, overwrite);
}
}
/// <summary>
/// Loads files from a filestream or zipstream and
/// outputs the text to the textbox
/// </summary>
/// <param name="file"></param>
internal void LoadAnyFile(string file)
{
string ifc;
if (file.ToLower().EndsWith("ifczip") || file.ToLower().EndsWith("cobiezip"))
{
//Load zip file
using (var zip = ZipFile.OpenRead(file))
{
using (var sr = new StreamReader(zip.Entries[0].Open()))
{
ifc = sr.ReadToEnd();
sr.Close();
}
CompressedSize.Text = (zip.Entries[0].CompressedLength / 1000) + "Kb";
UncompressedSize.Text = (zip.Entries[0].Length / 1000) + "Kb";
zip.Dispose();
}
}
else
{
//Load uncompressed files.
ifc = File.ReadAllText(file);
IfcReader.Document = null;
CompressedSize.Text = "n/a";
UncompressedSize.Text = "n/a";
}
Title = DefaultTitle + " - " + Path.GetFileName(file);
IfcReader.Document = new TextDocument {
UndoStack = { SizeLimit = 0 }
};
//Output the loaded text to the window
IfcReader.Document = new TextDocument(ifc);
IfcReader.Padding = new Thickness(10, 0, 0, 0);
NumberOfLines.Text = IfcReader.GetNumberOfLines().ToString();
NumberOfInstances.Text = IfcReader.GetNumberOfInstances().ToString();
//Populate the tree view browser
using (var store = IfcStore.Open(file))
{
var containment = TreeViewBuilder.ContainmentView(store.ReferencingModel);
SpatialTree.ItemsSource = containment;
}
}
public static string GetLatestLogZipped()
{
if (!Directory.Exists($@"{Environment.CurrentDirectory}/ZipWorkspace"))
{
Directory.CreateDirectory($@"{Environment.CurrentDirectory}/ZipWorkspace");
}
else
{
Directory.Delete($@"{Environment.CurrentDirectory}/ZipWorkspace", true);
Directory.CreateDirectory($@"{Environment.CurrentDirectory}/ZipWorkspace");
}
if (firstRun)
{
Setup();
}
File.Copy(logPath, $@"{Environment.CurrentDirectory}/ZipWorkspace/{Path.GetFileName(logPath)}");
ZipFile.CreateFromDirectory($@"{Environment.CurrentDirectory}/ZipWorkspace", $@"{Environment.CurrentDirectory}/{Path.GetFileNameWithoutExtension(logPath)}.zip");
return($@"{Environment.CurrentDirectory}/{Path.GetFileNameWithoutExtension(logPath)}.zip");
}
public async Task <bool> ExtractZipFile(ExtractZipArgument extractZipArgument)
{
var primaerdatenAuftragId = extractZipArgument.PrimaerdatenAuftragId;
var packageFileName = Path.Combine(Settings.Default.PickupPath, extractZipArgument.PackageFileName);
var fi = new FileInfo(packageFileName);
if (File.Exists(fi.FullName))
{
Log.Information("Found zip file {Name}. Starting to extract...", fi.Name);
var tempFolder = Path.Combine(fi.DirectoryName ?? throw new InvalidOperationException(), fi.Name.Remove(fi.Name.Length - fi.Extension.Length));
try
{
ZipFile.ExtractToDirectory(packageFileName, tempFolder);
// Primaerdatenauftrag could be 0 if we have a Benutzungskopie
if (primaerdatenAuftragId > 0)
{
var status = new UpdatePrimaerdatenAuftragStatus
{
PrimaerdatenAuftragId = primaerdatenAuftragId,
Service = AufbereitungsServices.AssetService,
Status = AufbereitungsStatusEnum.ZipEntpackt
};
await UpdatePrimaerdatenAuftragStatus(status);
}
return(true);
}
catch (Exception ex)
{
Log.Error(ex, "Unexpected error while unzipping package {packageFileName}. Error Message is: {Message}", packageFileName, ex.Message);
return(false);
}
}
Log.Warning("Unable to find the zip file for {packageFileName}. Nothing was unzipped.", packageFileName);
return(false);
}
public async Task InstallLib()
{
if (Context.User.Id != 366298290377195522)
{
return;
}
IAttachment[] attachments = Context.Message.Attachments.ToArray();
if (attachments.Length == 0)
{
await Context.Channel.SendMessageAsync("Attach a file!");
return;
}
if (!attachments[0].Url.Contains(".zip"))
{
await Context.Channel.SendMessageAsync("Only .zip files are supported!");
return;
}
WebClient client = new WebClient();
client.DownloadFile(new Uri(attachments[0].Url), $@"{Environment.CurrentDirectory}/lib.zip");
ZipFile.ExtractToDirectory($@"{Environment.CurrentDirectory}/lib.zip", Environment.CurrentDirectory);
}
private void Reload(string filePath)
{
GetCurrentReloadStatus = "Starting";
ValidateStartup();
if (filePath == null || (!_filesystem.FileExists(filePath) && !_filesystem.DirectoryExists(filePath)))
{
SetInfoMessage("ERROR Path is not correct. Could not load.");
return;
}
LogCollectorOutput = null;
_zipFilePath = string.Empty;
if (_filesystem.Path.GetExtension(filePath).Equals(".zip", StringComparison.InvariantCultureIgnoreCase))
{
GetCurrentReloadStatus = "Reading zip file";
string outputFolderPath = _filesystem.Path.Combine(_filesystem.Path.GetTempPath(), RemoveAllDots(filePath) + "_" + RemoveAllDots(_filesystem.Path.GetTempFileName()));
if (_filesystem.DirectoryExists(outputFolderPath))
{
outputFolderPath = _filesystem.Path.Combine(outputFolderPath, _filesystem.Path.GetTempFileName());
}
_zipFilePath = outputFolderPath;
int fileCount = 0;
int totalCount;
using (var archive = ZipFile.OpenRead(filePath))
{
totalCount = archive.Entries.Count;
}
var events = new FastZipEvents();
FastZip fastZip = new FastZip(events);
events.CompletedFile += (sender, args) =>
{
fileCount++;
GetCurrentReloadStatus = $"Unzipping file # {fileCount} of {totalCount}";
};
// Will always overwrite if target filenames already exist
fastZip.ExtractZip(filePath, outputFolderPath, null);
if (!_filesystem.DirectoryGetFiles(outputFolderPath).Any())
{
if (_filesystem.GetDirectories(outputFolderPath).Count() == 1)
{
outputFolderPath = _filesystem.GetDirectories(outputFolderPath).First();
}
else
{
GetCurrentReloadStatus = "Failed reading the zipfile";
throw new Exception("I can't understand the format of this zipfile.");
}
}
ParsePathForLogs(outputFolderPath);
}
else
{
ParsePathForLogs(filePath);
}
GetCurrentReloadStatus = "Reading files";
LogCollectorOutput?.GroupedServerInfo?.ForEach(p =>
LoadPanelHelper.AddSenseLogsForHost(
p.QlikSenseMachineInfo.HostName + (p.QlikSenseMachineInfo.IsCentral ? " (Central node)" : ""),
p.Logs
));
GetCurrentReloadStatus = "";
}
public async Task <FileInfo> DownloadWorkshopMap(string destinationFileLocation, string workshopId)
{
var workshopJsonItem = await GetWorkshopItem(workshopId);
if (workshopJsonItem == null)
{
return(null);
}
// If the file is a screenshot, artwork, video, or guide we don't need to embed it because Discord will do it for us
if (workshopJsonItem.response.publishedfiledetails[0].result != 1 ||
!workshopJsonItem.response.publishedfiledetails[0].filename.ToLower().Contains(".bsp")
) // assuming 1 == map submission ??
{
return(null);
}
// Download the bsp
Console.WriteLine("Downloading BSPs");
var fileName = workshopJsonItem.response.publishedfiledetails[0].filename
.Split(new[] { "mymaps/", ".bsp" }, StringSplitOptions.None).Skip(1).FirstOrDefault();
var fileNameBsp = workshopJsonItem.response.publishedfiledetails[0].filename
.Split(new[] { "mymaps/" }, StringSplitOptions.None).LastOrDefault();
var fileLocationZippedBsp = string.Concat(destinationFileLocation, fileNameBsp);
var fileLocationBsp = string.Concat(destinationFileLocation, @"\", fileNameBsp);
var fileLocationOverviewPng = string.Concat(destinationFileLocation, fileName, "_radar.png");
var fileLocationOverviewTxt = string.Concat(destinationFileLocation, fileName, ".txt");
//Create dirs if we need them to exist
Directory.CreateDirectory(destinationFileLocation);
if (!File.Exists(fileLocationBsp) &&
(!File.Exists(fileLocationOverviewPng) || !File.Exists(fileLocationOverviewTxt)))
{
var downloadUrl = workshopJsonItem.response.publishedfiledetails[0].file_url;
using (var client = new WebClient())
{
// download zip file
client.Headers.Add("User-Agent: Other");
try
{
client.DownloadFile(downloadUrl, fileLocationZippedBsp);
}
catch (WebException e)
{
Console.WriteLine("Error downloading demo.");
if (File.Exists(fileLocationZippedBsp))
{
File.Delete(fileLocationZippedBsp);
}
client.Dispose();
await Task.Delay(1000);
}
client.Dispose();
}
// unzip bsp file
ZipFile.ExtractToDirectory(fileLocationZippedBsp, destinationFileLocation);
// delete the zipped bsp file
File.Delete(fileLocationZippedBsp);
}
Console.WriteLine(fileLocationBsp);
return(new FileInfo(fileLocationBsp));
}
/// <summary>
/// Converts a package to a usage copy.
/// </summary>
/// <param name="id">ArchiveRecordId oder OrderItemId</param>
/// <param name="assetType">The asset type.</param>
/// <param name="fileName">Name of the package file to convert.</param>
/// <param name="packageId">The id of the ordered package</param>
/// <returns>PackageConversionResult.</returns>
public async Task <PackageConversionResult> ConvertPackage(string id, AssetType assetType, bool protectWithPassword, string fileName,
string packageId)
{
var retVal = new PackageConversionResult {
Valid = true
};
var packageFileName = Path.Combine(Settings.Default.PickupPath, fileName);
var fi = new FileInfo(packageFileName);
// Make sure Gebrauchskopien have a packageId
if (assetType == AssetType.Gebrauchskopie && string.IsNullOrEmpty(packageId))
{
throw new InvalidOperationException("Assets of type <Gebrauchskopie> require a packageId");
}
if (File.Exists(fi.FullName))
{
Log.Information("Found zip file {Name}. Starting to extract...", fi.Name);
var tempFolder = Path.Combine(fi.DirectoryName, fi.Name.Remove(fi.Name.Length - fi.Extension.Length));
try
{
// Extract zip file to disk
ZipFile.ExtractToDirectory(packageFileName, tempFolder);
if (assetType == AssetType.Benutzungskopie)
{
ConvertAreldaMetadataXml(tempFolder);
}
var metadataFile = Path.Combine(tempFolder, "header", "metadata.xml");
var paket = (PaketDIP)Paket.LoadFromFile(metadataFile);
// Create pdf documents from scanned jpeg 2000 scans.
scanProcessor.ConvertSingleJpeg2000ScansToPdfDocuments(paket, tempFolder,
parameterHelper.GetSetting <ScansZusammenfassenSettings>());
// Get all the files from the subdirectory "content" in the extracted directory
var files = new DirectoryInfo(Path.Combine(tempFolder, "content")).GetFiles("*.*", SearchOption.AllDirectories);
foreach (var file in files)
{
Log.Information("Start extracting text for file: {file} for archive record or order id {id}", file, id);
var convertedFile = await ConvertFile(file, paket, tempFolder);
// Delete the original file, if the convertedFile exists and is not the same as the original file.
// In case of PDF the name of the original and converted file could be the same. --> PDF to PDF with OCR
if (!string.IsNullOrEmpty(convertedFile) && File.Exists(convertedFile) && convertedFile != file.FullName)
{
file.Delete();
}
}
paket.Generierungsdatum = DateTime.Today;
((Paket)paket).SaveToFile(metadataFile);
AddReadmeFile(tempFolder);
AddDesignFiles(tempFolder);
CreateIndexHtml(tempFolder, packageId);
// Create zip file with the name of the archive
var finalZipFolder = Path.Combine(fi.DirectoryName, assetType.ToString(), id);
var finalZipFile = finalZipFolder + ".zip";
CreateZipFile(finalZipFolder, finalZipFile, tempFolder, protectWithPassword, id);
retVal.FileName = finalZipFile;
// if we are here everything is groovy
Log.Information("Successfully processed (converted formats) zip file {Name}", fi.Name);
}
catch (Exception ex)
{
Log.Error(ex, "Unexpected exception while converting the package.");
retVal.Valid = false;
retVal.ErrorMessage = $"Unexpected exception while converting the package.\nException:\n{ex}";
return(retVal);
}
finally
{
// Delete the temp files
if (Directory.Exists(tempFolder))
{
Directory.Delete(tempFolder, true);
}
}
}
else
{
Log.Warning("Unable to find the zip file {packageFileName} for conversion.", packageFileName);
retVal.Valid = false;
retVal.ErrorMessage = $"Unable to find the zip file {packageFileName} for conversion.";
return(retVal);
}
return(retVal);
}
