public void SwapVhdxGzip()
{
SetupHelper.SetupComplete();
using (FileStream fs = File.OpenRead(Path.Combine("..", "..", "..", "Swap", "Data", "swap.zip")))
using (Stream vhdx = ZipUtilities.ReadFileFromZip(fs))
using (var diskImage = new DiskImageFile(vhdx, Ownership.Dispose))
using (var disk = new Disk(new List <DiskImageFile> {
diskImage
}, Ownership.Dispose))
{
var manager = new VolumeManager(disk);
var logicalVolumes = manager.GetLogicalVolumes();
Assert.Equal(1, logicalVolumes.Length);
var volume = logicalVolumes[0];
var filesystems = FileSystemManager.DetectFileSystems(volume);
Assert.Equal(1, filesystems.Length);
var filesystem = filesystems[0];
Assert.Equal("Swap", filesystem.Name);
var swap = filesystem.Open(volume);
Assert.IsType <SwapFileSystem>(swap);
Assert.Equal(0, swap.AvailableSpace);
Assert.Equal(10737414144, swap.Size);
Assert.Equal(swap.Size, swap.UsedSpace);
}
}
private bool ZipReports(string zipPath, string[] outputFiles)
{
executor.Invoke("Information: Zipping reports for filing package.");
string rootDirectoryName = string.Empty;
for (int i = 0; i < outputFiles.Length; i++)
{
if (string.IsNullOrEmpty(rootDirectoryName))
{
rootDirectoryName = Path.GetDirectoryName(outputFiles[i]);
outputFiles[i] = Path.GetFileName(outputFiles[i]);
}
else if (!outputFiles[i].StartsWith(rootDirectoryName))
{
executor.Invoke("Error: Zip creation failed.");
return(false);
}
else
{
outputFiles[i] = Path.GetFileName(outputFiles[i]);
}
}
if (ZipUtilities.TryZipAndCompressFiles(zipPath, rootDirectoryName, outputFiles))
{
executor.Invoke("Information: Zip successfully created." + separator + "\tLocation: " + zipPath);
return(true);
}
else
{
executor.Invoke("Error: Zip creation failed.");
return(false);
}
}
/// <summary>
/// Download and execute a file from a url.
/// </summary>
/// <param name="environmentFolder">The folder where the file should be downloaded to</param>
/// <param name="url">The URL to get the file from</param>
/// <param name="commandLineArguments">Any command line arguments to use when running file</param>
/// <param name="downloadedFileName">The name of the file to download</param>
private static void DownloadAndExecuteFile(string environmentFolder, string url, string commandLineArguments, string downloadedFileName)
{
Directory.CreateDirectory(Path.GetDirectoryName(downloadedFileName));
if (url != null)
{
WebClient myWebClient = new WebClient();
byte[] bytes = myWebClient.DownloadData(url);
using (FileStream writer = File.Create(downloadedFileName))
writer.Write(bytes, 0, bytes.Length);
}
if (Path.GetExtension(downloadedFileName) == ".exe")
{
ProcessStartInfo startInfo = new ProcessStartInfo
{
FileName = downloadedFileName,
Arguments = commandLineArguments,
WorkingDirectory = environmentFolder,
UseShellExecute = true,
CreateNoWindow = true
};
Process p = Process.Start(startInfo);
p.WaitForExit();
}
else
{
ZipUtilities.UnZipFiles(downloadedFileName, environmentFolder, null);
}
File.Delete(downloadedFileName);
}
public static Task <ExitCode> CreateReport(ExcelReportOptions opts, TextWriter stdErr)
{
if (!File.Exists(opts.PathToHealthExportFile))
{
return(Task.FromResult(ExitCode.ExportFileNotFound(opts.PathToHealthExportFile)));
}
if (File.Exists(opts.OutputFilename))
{
return(Task.FromResult(ExitCode.ExportFileExists(opts.OutputFilename)));
}
var loader = Usable.Using(new StreamReader(opts.PathToHealthExportFile), reader =>
ZipUtilities.ReadArchive(
reader.BaseStream,
entry => entry.FullName == "apple_health_export/export.xml",
entry => new XmlReaderExportLoader(entry.Open()))
.FirstOrDefault());
var settings = GetSettings(opts, stdErr);
var(package, customSheets) = GetCustomSheets(opts, stdErr);
using (var excelFile = new ExcelPackage())
using (package)
{
var timeZone = DateTimeZone.ForOffset(Offset.FromHours(-5));
ExcelReport.BuildReport(loader.Records, loader.Workouts, excelFile.Workbook, settings, timeZone, customSheets);
excelFile.SaveAs(new FileInfo(opts.OutputFilename));
}
return(Task.FromResult(ExitCode.Success));
}
public void XfsVhdxZip()
{
SetupHelper.SetupComplete();
using (FileStream fs = File.OpenRead(Path.Combine("..", "..", "..", "Xfs", "Data", "xfs.zip")))
using (Stream vhdx = ZipUtilities.ReadFileFromZip(fs))
using (var diskImage = new DiskImageFile(vhdx, Ownership.Dispose))
using (var disk = new Disk(new List <DiskImageFile> {
diskImage
}, Ownership.Dispose))
{
var manager = new VolumeManager(disk);
var logicalVolumes = manager.GetLogicalVolumes();
Assert.Equal(1, logicalVolumes.Length);
var volume = logicalVolumes[0];
var filesystems = FileSystemManager.DetectFileSystems(volume);
Assert.Equal(1, filesystems.Length);
var filesystem = filesystems[0];
Assert.Equal("xfs", filesystem.Name);
var xfs = filesystem.Open(volume);
Assert.IsType <XfsFileSystem>(xfs);
Assert.Equal(9082019840, xfs.AvailableSpace);
Assert.Equal(10725863424, xfs.Size);
Assert.Equal(1643843584, xfs.UsedSpace);
ValidateContent(xfs);
}
}
public void AddNoFileToArchiveFails()
{
string zipPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".zip");
Assert.IsFalse(ZipUtilities.AddFileToArchive(zipPath, null));
Assert.IsFalse(ZipUtilities.AddFileToArchive(zipPath, string.Empty));
}
public void AddSingleFileToNonExistingZipSuccessTest()
{
string ticks = $"{DateTime.Now.Ticks}";
string zipName = $"arc{ticks}.zip";
string zipPath = $"{Path.Combine(Path.GetTempPath(), zipName)}";
// Create file with some data
string fileName = $"file{ticks}.txt";
string filePath = Path.Combine(Path.GetTempPath(), fileName);
File.WriteAllText(filePath, $"{this.GetType()}");
Assert.IsTrue(File.Exists(filePath));
// Add the file to the zip
bool result = ZipUtilities.AddFilesToArchive(zipPath, new List <string>()
{
filePath
});
Assert.IsTrue(result);
Assert.IsTrue(File.Exists(zipPath));
// Cleanup
File.Delete(filePath);
File.Delete(zipPath);
}
static void Main(string[] args)
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
var fileLocation = @"c:\users\jcfuller\Downloads\export.zip";
XDocument export = null;
using (var reader = new StreamReader(fileLocation))
{
export = ZipUtilities.ReadArchive(
reader.BaseStream,
entry => entry.FullName == "apple_health_export/export.xml",
entry => XDocument.Load(entry.Open()))
.FirstOrDefault();
}
var settings = Settings.Default;
settings.UseConstantNameForMostRecentMonthlySummarySheet = true;
settings.UseConstantNameForPreviousMonthlySummarySheet = true;
using (var excelFile = new ExcelPackage())
{
ExcelReport.BuildReport(export, excelFile.Workbook, settings, Enumerable.Empty <ExcelWorksheet>());
excelFile.SaveAs(new FileInfo(@"c:\users\jcfuller\Desktop\test-edt.xlsx"));
}
Console.WriteLine("done, press a key");
Console.ReadKey();
}
public void ExtractArchiveToDirectoryTest()
{
string ticks = $"{DateTime.Now.Ticks}";
string directoryName = $"dir{ticks}";
string directoryPath = Path.Combine(Path.GetTempPath(), directoryName);
string zipPath = $"{directoryPath}.zip";
// Create empty directory
Directory.CreateDirectory(directoryPath);
Assert.IsTrue(Directory.Exists(directoryPath));
// Zip empty directory
ZipFile.CreateFromDirectory(directoryPath, zipPath);
Assert.IsTrue(File.Exists(zipPath));
// Create file with some data
string fileName = $"file{ticks}.txt";
string filePath = Path.Combine(Path.GetTempPath(), fileName);
File.WriteAllText(filePath, $"{this.GetType()}");
Assert.IsTrue(File.Exists(filePath));
// Get initial zip size
FileInfo fileInfo = new FileInfo(zipPath);
var initialZipSize = fileInfo.Length;
// Add the file to the zip
bool result = ZipUtilities.AddFilesToArchive(zipPath, new List <string>()
{
filePath
});
Assert.IsTrue(result);
// Get new zip size
fileInfo.Refresh();
var newZipSize = fileInfo.Length;
// Expect new size is bigger than initial zip size
Assert.IsTrue(newZipSize > initialZipSize);
string outputDir = Path.Combine(Path.GetTempPath(), $"output{ticks}");
var extractionResult = ZipUtilities.ExtractArchive(zipPath, outputDir, false);
Assert.IsTrue(extractionResult);
Assert.IsTrue(Directory.Exists(outputDir));
extractionResult = ZipUtilities.ExtractArchive(zipPath, outputDir, true);
Assert.IsTrue(extractionResult);
// Cleanup
Directory.Delete(outputDir, true);
Directory.Delete(directoryPath);
File.Delete(filePath);
File.Delete(zipPath);
}
/// <summary>Zips the files.</summary>
/// <param name="intoFileName">The name of the file to create.</param>
/// <param name="fileNames">The file names to zip.</param>
private static void ZipFiles(string[] fileNames, string intoFileName)
{
// Zip up files.
ZipUtilities.ZipFiles(fileNames, null, intoFileName);
// Delete the .met files.
foreach (string fileName in fileNames)
{
File.Delete(fileName);
}
}
public void BtrfsVhdxZip()
{
DiscUtils.Setup.SetupHelper.RegisterAssembly(typeof(Disk).GetTypeInfo().Assembly);
DiscUtils.Setup.SetupHelper.RegisterAssembly(typeof(BtrfsFileSystem).GetTypeInfo().Assembly);
using (FileStream fs = File.OpenRead(Path.Combine("..", "..", "..", "Btrfs", "Data", "btrfs.zip")))
using (Stream vhdx = ZipUtilities.ReadFileFromZip(fs))
using (var diskImage = new DiskImageFile(vhdx, Ownership.Dispose))
using (var disk = new Disk(new List <DiskImageFile> {
diskImage
}, Ownership.Dispose))
{
var manager = new VolumeManager(disk);
var logicalVolumes = manager.GetLogicalVolumes();
Assert.Equal(1, logicalVolumes.Length);
var volume = logicalVolumes[0];
var filesystems = FileSystemManager.DetectFileSystems(volume);
Assert.Equal(1, filesystems.Length);
var filesystem = filesystems[0];
Assert.Equal("btrfs", filesystem.Name);
using (var btrfs = filesystem.Open(volume))
{
Assert.IsType <BtrfsFileSystem>(btrfs);
Assert.Equal(1072594944, btrfs.AvailableSpace);
Assert.Equal(1072693248, btrfs.Size);
Assert.Equal(98304, btrfs.UsedSpace);
var subvolumes = ((BtrfsFileSystem)btrfs).GetSubvolumes();
Assert.Equal(1, subvolumes.Length);
Assert.Equal(256UL, subvolumes[0].Id);
Assert.Equal("subvolume", subvolumes[0].Name);
Assert.Equal("text\n", GetFileContent(@"\folder\subfolder\file", btrfs));
Assert.Equal("f64464c2024778f347277de6fa26fe87", GetFileChecksum(@"\folder\subfolder\f64464c2024778f347277de6fa26fe87", btrfs));
Assert.Equal("fa121c8b73cf3b01a4840b1041b35e9f", GetFileChecksum(@"\folder\subfolder\fa121c8b73cf3b01a4840b1041b35e9f", btrfs));
IsAllZero(@"folder\subfolder\sparse", btrfs);
Assert.Equal("test\n", GetFileContent(@"\subvolume\subvolumefolder\subvolumefile", btrfs));
Assert.Equal("b0d5fae237588b6641f974459404d197", GetFileChecksum(@"\folder\subfolder\compressed", btrfs));
Assert.Equal("test\n", GetFileContent(@"\folder\symlink", btrfs)); //PR#36
Assert.Equal("b0d5fae237588b6641f974459404d197", GetFileChecksum(@"\folder\subfolder\lzo", btrfs));
}
using (var subvolume = new BtrfsFileSystem(volume.Open(), new BtrfsFileSystemOptions {
SubvolumeId = 256, VerifyChecksums = true
}))
{
Assert.Equal("test\n", GetFileContent(@"\subvolumefolder\subvolumefile", subvolume));
}
}
}
public void ReplayLog()
{
using (FileStream fs = File.OpenRead(Path.Combine("..", "..", "..", "Vhdx", "Data", "vhdx-log-replay.zip")))
using (Stream vhdx = ZipUtilities.ReadFileFromZip(fs))
using (var diskImage = new DiskImageFile(vhdx, Ownership.Dispose))
using (var disk = new Disk(new List <DiskImageFile> {
diskImage
}, Ownership.Dispose))
{
Assert.True(disk.IsPartitioned);
Assert.Equal(2, disk.Partitions.Count);
}
}
public static Submission MapFrom(SubmissionDto submissionDto)
{
return(new Submission
{
SubmitterId = submissionDto.SubmitterId,
SubmissionId = submissionDto.SubmissionId,
ApplicationMode = submissionDto.ApplicationMode,
ReferenceSolution = ZipUtilities.UnZipToMemory(submissionDto.ReferenceSolution),
ReferenceTestSolution = ZipUtilities.UnZipToMemory(submissionDto.ReferenceTestSolution),
PackageName = submissionDto.SolutionFolderName,
AssignmentSolution = ZipUtilities.UnZipToMemory(submissionDto.AssignmentSolution),
TestCaseSolution = ZipUtilities.UnZipToMemory(submissionDto.TestCaseSolution)
});
}
private static bool UnzipPackage(Filing f, string zipPath, out string[] zipFiles)
{
zipFiles = new string[0];
string errorMsg;
if (!ZipUtilities.TryUnzipAndUncompressFiles(f.InstancePath, zipPath, out zipFiles, out errorMsg))
{
Console.WriteLine("Error: The ZIP file cannot be opened.");
Console.WriteLine("\tSkipping filing: " + f.InstancePath);
return(false);
}
return(true);
}
private static bool UnzipPackage(UpdateStatusExecutor executor, Filing f, string zipPath, out string[] zipFiles)
{
zipFiles = new string[0];
string errorMsg;
if (!ZipUtilities.TryUnzipAndUncompressFiles(f.InstancePath, zipPath, out zipFiles, out errorMsg))
{
executor.Invoke("Error: The ZIP file cannot be opened.");
executor.Invoke("\tSkipping filing: " + f.InstancePath);
return(false);
}
return(true);
}
/// <summary>
/// Extract the contents of a zip file located in the specified path to a temp directory
/// </summary>
/// <returns>path of the temporary directory</returns>
public static string ExtractToTempDirectory(string zipFilePath)
{
if (!File.Exists(zipFilePath))
{
throw new ArgumentException("Zip file must exist.", "zipFilePath");
}
string tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(tempPath);
ZipUtilities.ExtractToDirectory(zipFilePath, tempPath);
return(tempPath);
}
private void ZipFiles(DirectoryInfo directory, string zipName)
{
output op = new output(compare.Output);
string[] fileNames = BuildString(directory.GetFiles());
try
{
ZipUtilities.TryZipAndCompressFiles(zipName, directory.FullName, fileNames);
}
catch (Exception ex)
{
op(string.Format(Program.zipCopyErr, zipName, ex.Message));
}
}
private bool UnzipMove(DirectoryInfo directory, out string[] files, out string error)
{
//Directory passed should be \Reports
bool retval = true;
FileInfo[] zipped = directory.GetFiles("*.zip");
files = null;
error = string.Empty;
string fileName = string.Empty;
try
{
foreach (FileInfo zip in zipped)
{
fileName = zip.Name;
fileName = fileName.Remove(fileName.Length - 4);
ZipUtilities.TryUnzipAndUncompressFiles(zip.FullName,
zip.Directory.Parent.FullName + "\\Comparison\\" + fileName,
out files, out error);
}
}
catch (System.Security.SecurityException ex)
{
error = "Failed proccess report with the following exception: " + ex.Message;
RLogger.Error(error, ex);
retval = false;
}
catch (UnauthorizedAccessException ex)
{
error = "Failed proccess report with the following exception: " + ex.Message;
RLogger.Error(error, ex);
retval = false;
}
catch (PathTooLongException ex)
{
error = "Failed proccess report with the following exception: " + ex.Message;
RLogger.Error(error, ex);
retval = false;
}
catch (Exception ex)
{
error = "Failed proccess report with the following exception: " + ex.Message;
RLogger.Error(error, ex);
retval = false;
}
return(retval);
}
public void AppleTestZip()
{
using (FileStream fs = File.OpenRead(Path.Combine("..", "..", "..", "Iso9660", "Data", "apple-test.zip")))
using (Stream iso = ZipUtilities.ReadFileFromZip(fs))
using (CDReader cr = new CDReader(iso, false))
{
DiscDirectoryInfo dir = cr.GetDirectoryInfo("sub-directory");
Assert.NotNull(dir);
Assert.Equal("sub-directory", dir.Name);
DiscFileInfo[] file = dir.GetFiles("apple-test.txt");
Assert.Equal(1, file.Length);
Assert.Equal(21, file[0].Length);
Assert.Equal("apple-test.txt", file[0].Name);
Assert.Equal(dir, file[0].Directory);
}
}
/// <summary>Called when all jobs completed</summary>
public void Completed()
{
// Look for an error log file. Seems the error file gets written to the AusFarm
// directory rather than the same place as the .sdml.
Errors = new List <string>();
foreach (string errorFile in Directory.GetFiles(binFolder, "*_errors.log"))
{
Errors.Add(File.ReadAllText(errorFile));
File.Delete(errorFile);
}
// Zip the temporary directory
AllFilesZipped = new MemoryStream();
ZipUtilities.ZipFiles(Directory.GetFiles(workingDirectory), null, AllFilesZipped);
// Get rid of our temporary directory.
Directory.Delete(workingDirectory, true);
}
public IEnumerable <DataDetail> Upload(DataFile dataFile)
{
Check.IsNotNull(dataFile, "dataFile");
List <DataDetail> collection = new List <DataDetail>();
List <DataFile> dataFiles = null;
using (MemoryStream memoryStream = new MemoryStream(dataFile.FileContent))
{
dataFiles = ZipUtilities.GetListOfFilesFromStream(memoryStream, this.userId);
}
foreach (var df in dataFiles)
{
var uploadedDataFiles = base.Upload(df);
collection.Add(uploadedDataFiles);
}
return(collection);
}
public void Lvm2VhdxZip()
{
SetupHelper.SetupComplete();
using (FileStream fs = File.OpenRead(Path.Combine("..", "..", "..", "Lvm", "Data", "lvm2.zip")))
using (Stream vhdx = ZipUtilities.ReadFileFromZip(fs))
using (var diskImage = new DiskImageFile(vhdx, Ownership.Dispose))
using (var disk = new Disk(new List <DiskImageFile> {
diskImage
}, Ownership.Dispose))
{
var manager = new VolumeManager(disk);
var logicalVolumes = manager.GetLogicalVolumes();
Assert.Equal(3, logicalVolumes.Length);
Assert.Equal(1283457024, logicalVolumes[0].Length);
Assert.Equal(746586112, logicalVolumes[1].Length);
Assert.Equal(1178599424, logicalVolumes[2].Length);
}
}
/// <summary>Called when all jobs completed</summary>
public void Completed()
{
// Perform cleanup and get outputs.
Outputs = PerformCleanup(WorkingDirectory);
// Look for error files - apsimx produces these.
Errors = new List <string>();
foreach (string errorFile in Directory.GetFiles(WorkingDirectory, "*.error"))
{
string error = string.Empty;
foreach (string line in File.ReadAllLines(errorFile))
{
if (!line.StartsWith("File:") && !line.StartsWith("Finished running simulations"))
{
error += line;
}
}
if (error != string.Empty)
{
Errors.Add(error);
}
}
// Look for error table - apsim classic produces these.
if (Outputs.Tables.Contains("Error"))
{
DataTable errorTable = Outputs.Tables["Error"];
foreach (DataRow errorRow in errorTable.Rows)
{
Errors.Add(errorRow["Text"].ToString());
}
}
// Zip the temporary directory
AllFilesZipped = new MemoryStream();
ZipUtilities.ZipFiles(Directory.GetFiles(WorkingDirectory), null, AllFilesZipped);
// Get rid of our temporary directory.
Directory.Delete(WorkingDirectory, true);
}
/// <summary>
/// Creates a instance of a yield prophet spec from a zip file.
/// </summary>
/// <param name="zipFileName">The name of the .zip file.</param>
/// <returns>The newly create yieldprophet object.</returns>
private static YieldProphet YieldProphetFromZip(string zipFileName)
{
YieldProphet yieldProphet;
string tempFolder = Path.GetTempFileName();
File.Delete(tempFolder);
Directory.CreateDirectory(tempFolder);
FileStream reader = File.OpenRead(zipFileName);
try
{
string[] fileNames = ZipUtilities.UnZipFiles(reader, tempFolder, null);
string fileName = Path.Combine(tempFolder, "YieldProphet.xml");
if (!File.Exists(fileName))
{
// Look for first XML file.
foreach (string file in fileNames)
{
if (file.Contains(".xml"))
{
fileName = file;
break;
}
}
}
yieldProphet = YieldProphetUtility.YieldProphetFromFile(fileName);
yieldProphet.ReportName = Path.GetFileNameWithoutExtension(fileName);
}
finally
{
reader.Close();
}
Directory.Delete(tempFolder, true);
return(yieldProphet);
}
public ActionResult Execute(ArgumentCollection arguments)
{
string outputDirPath = null;
string sourceZipPath = null;
try
{
if (arguments.HasArgument(UnzipArchiveActionExecutionArgs.OutputDirPath))
{
outputDirPath = arguments.GetValue <string>(UnzipArchiveActionExecutionArgs.OutputDirPath);
}
if (arguments.HasArgument(UnzipArchiveActionExecutionArgs.SourceZipPath))
{
sourceZipPath = arguments.GetValue <string>(UnzipArchiveActionExecutionArgs.SourceZipPath);
}
if (string.IsNullOrWhiteSpace(outputDirPath))
{
if (!string.IsNullOrWhiteSpace(OutputDirPath))
{
outputDirPath = OutputDirPath;
}
}
if (string.IsNullOrWhiteSpace(sourceZipPath))
{
if (!string.IsNullOrWhiteSpace(SourceZipPath))
{
sourceZipPath = SourceZipPath;
}
}
if (string.IsNullOrWhiteSpace(sourceZipPath))
{
throw new MissingArgumentException(UnzipArchiveActionExecutionArgs.SourceZipPath);
}
if (string.IsNullOrWhiteSpace(outputDirPath))
{
throw new MissingArgumentException(UnzipArchiveActionExecutionArgs.OutputDirPath);
}
var result = ZipUtilities.ExtractArchive(sourceZipPath, outputDirPath, OverrideFiles);
return(result
? ActionResult.Succeeded().WithAdditionInformation(ArgumentCollection.New()
.WithArgument(UnzipArchiveActionResultsArgs.OutputDirPath, outputDirPath)
.WithArgument(UnzipArchiveActionResultsArgs.SourceZipPath, sourceZipPath)
)
: ActionResult.Failed().WithAdditionInformation(ArgumentCollection.New()
.WithArgument(UnzipArchiveActionResultsArgs.OutputDirPath, outputDirPath)
.WithArgument(UnzipArchiveActionResultsArgs.SourceZipPath, sourceZipPath)));
}
catch (Exception exception)
{
return(ActionResult.Failed().WithException(exception).WithAdditionInformation(ArgumentCollection.New()
.WithArgument(UnzipArchiveActionResultsArgs.OutputDirPath, outputDirPath)
.WithArgument(UnzipArchiveActionResultsArgs.SourceZipPath, sourceZipPath)));
}
}
public bool TryMoveFilingsToProcessingFolder(out List <FilingInfo> filings)
{
filings = new List <FilingInfo>();
FileInfo[] zipFiles = FilingProcessorManager.TheMgr.FilingsFolderInfo.GetFiles("*.zip", SearchOption.TopDirectoryOnly);
Array.Sort(zipFiles, (left, right) => DateTime.Compare(left.CreationTime, right.CreationTime));
Guid batchId = Guid.NewGuid();
DirectoryInfo processingBatchFolder = FilingProcessorManager.TheMgr.ProcessingFolderInfo.CreateSubdirectory(batchId.ToString());
foreach (FileInfo zipFile in zipFiles)
{
string filingName = Path.GetFileNameWithoutExtension(zipFile.Name);
if (string.IsNullOrEmpty(filingName))
{
try
{
string errorFile = Path.Combine(FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, "UNKNOWN.txt");
string errorMessage = "Cannot process a filing zip which has no base name.";
FilingProcessorManager.TheMgr.WriteLogEntry(errorMessage, EventLogEntryType.Error);
File.WriteAllText(errorFile, errorMessage);
zipFile.Delete();
} catch { }
continue;
}
DirectoryInfo filingProcessingFolder = processingBatchFolder.CreateSubdirectory(filingName);
string[] unzippedFiles;
string filingZipFile = string.Format("{0}{1}{2}", filingProcessingFolder.FullName, Path.DirectorySeparatorChar, Path.GetFileName(zipFile.Name));
if (filingRetryCount.ContainsKey(filingName) &&
filingRetryCount[filingName] >= maxUnzipAttempts)
{
filingRetryCount.Remove(filingName);
//We have failed to extract the filing multiple times, stop trying to make it work,
//remove the zip file from the filing folder, log an error in the reports folder,
//and reomve the entry from the dictionary
string errorFilename = filingName + "_" + ERROR_FILE_NAME;
string errorFile = Path.Combine(FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, filingName);
string errorMsg = string.Format("Cannot extract files for filing {0}. The max number of retries has been reached, removing the zip file from the Filings folder. ", Path.GetFileNameWithoutExtension(zipFile.FullName));
if (File.Exists(errorFile))
{
FileUtilities.DeleteFile(new FileInfo(errorFile), true);
}
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
File.WriteAllText(errorFile, errorMsg);
zipFile.CopyTo(filingZipFile);
FileUtilities.DeleteFile(zipFile, true);
}
else
{
string zipError;
if (ZipUtilities.TryUnzipAndUncompressFiles(zipFile.FullName, filingProcessingFolder.FullName, out unzippedFiles, out zipError))
{
FilingInfo filing = new FilingInfo();
filing.AccessionNumber = filingProcessingFolder.Name;
filing.ParentFolder = processingBatchFolder.FullName;
filings.Add(filing);
zipFile.CopyTo(filingZipFile, true);
FileUtilities.DeleteFile(zipFile, true);
//SEC0145 - Only queue one filing at a time
break;
}
else
{
//Delete the folder from the processing batch folder sense it doesn't contain a valid filing
FileUtilities.DeleteDirectory(filingProcessingFolder, true, true);
if (!filingRetryCount.ContainsKey(filingName))
{
filingRetryCount.Add(filingName, 0);
}
filingRetryCount[filingName]++;
FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Can not extract files for filing {0}. The zip file may not be complete. ", Path.GetFileNameWithoutExtension(zipFile.FullName)), EventLogEntryType.Warning);
}
}
}
if (processingBatchFolder.GetDirectories().Length == 0)
{
//There were not any valid filings in this batch, remove the batch folder
FileUtilities.DeleteDirectory(processingBatchFolder, true, true);
}
return(true);
}
protected bool BuildReports(FilingInfo filing, string reportsFolder, out string error)
{
error = null;
bool foundFiles = true;
string errorMsg = string.Empty;
string filingPath = Path.Combine(filing.ParentFolder, filing.AccessionNumber);
string filingReportsPath = Path.Combine(filingPath, "Reports");
string filingErrorFile = Path.Combine(filingReportsPath, ERROR_FILE_NAME);
string instancePath = filing.GetInstanceDocPath();
string taxonomyPath = filing.GetTaxonomyPath();
if (string.IsNullOrEmpty(instancePath) ||
!File.Exists(instancePath))
{
errorMsg = string.Format("Can not find instance document for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
else if (string.IsNullOrEmpty(taxonomyPath) ||
!File.Exists(taxonomyPath))
{
errorMsg = string.Format("Can not find taxonomy file for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
bool buildSucceeded = false;
if (foundFiles)
{
string baseResourcePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Setting base path for Rules Engine. Path: {0}", baseResourcePath), EventLogEntryType.Information);
RulesEngineUtils.SetBaseResourcePath(baseResourcePath);
ReportBuilder.SetSynchronizedResources(true);
FilingProcessorManager.TheMgr.WriteLogEntry("Selected rule file: '" + FinancialRuleFile + "' for instance document: '" + Path.GetFileName(instancePath) + "'", EventLogEntryType.Information);
ReportBuilder rb = new ReportBuilder(FinancialRuleFile, ReportFormat, HtmlReportFormat);
rb.CurrencyMappingFile = CurrencyMappingFile;
rb.RemoteFileCachePolicy = RemoteFileCachePolicy;
//if( XmlCatalog != null )
// rb.XmlCatalog = XmlCatalog;
//Reports will be saves under the Filing's Processing folder then copied out to the actual reports folder,
//this will allow us to only copy complete sets of R files to the Reports Folder.
string filingSummaryFile = string.Format("{0}{1}{2}", filingReportsPath, Path.DirectorySeparatorChar, FilingSummary.FilingSummaryXmlName);
//Make sure there is a clean folder where the reports will be written to.
if (Directory.Exists(filingReportsPath))
{
Directory.Delete(filingReportsPath, true);
}
Directory.CreateDirectory(filingReportsPath);
FilingSummary summary = null;
buildSucceeded = rb.BuildReports(instancePath, taxonomyPath, filingSummaryFile, filingReportsPath, out summary, out error);
if (!buildSucceeded)
{
errorMsg = string.Format("Call to BuildReports failed for Filing {0}: {1}", filing.AccessionNumber, error);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
if (!Directory.Exists(filingReportsPath))
{
Directory.CreateDirectory(filingReportsPath);
}
File.WriteAllText(filingErrorFile, errorMsg);
}
}
else
{
if (!Directory.Exists(filingReportsPath))
{
Directory.CreateDirectory(filingReportsPath);
}
File.WriteAllText(filingErrorFile, errorMsg);
}
try
{
string errorFileName = filing.AccessionNumber + "_" + Path.GetFileName(filingErrorFile);
string reportErrorFile = Path.Combine(reportsFolder, errorFileName);
string reportZipFile = Path.Combine(reportsFolder, filing.AccessionNumber + ".zip");
if (File.Exists(reportErrorFile))
{
FileUtilities.DeleteFile(new FileInfo(reportErrorFile), true);
}
if (File.Exists(reportZipFile))
{
FileUtilities.DeleteFile(new FileInfo(reportZipFile), true);
}
if (buildSucceeded)
{
string[] filePathsToZip = Directory.GetFiles(filingReportsPath);
string[] filesToZip = new string[filePathsToZip.Length];
for (int i = 0; i < filesToZip.Length; i++)
{
filesToZip[i] = Path.GetFileName(filePathsToZip[i]);
}
string zipFile = Path.Combine(filingReportsPath, filing.AccessionNumber + ".zip");
if (ZipUtilities.TryZipAndCompressFiles(zipFile, filingReportsPath, filesToZip))
{
File.Copy(zipFile, reportZipFile);
}
}
else
{
File.Copy(filingErrorFile, reportErrorFile);
}
if (DeleteProcessedFilings)
{
DirectoryInfo di = new DirectoryInfo(filingPath);
FileUtilities.DeleteDirectory(di, true, true);
di = new DirectoryInfo(filing.ParentFolder);
if (di.GetDirectories().Length == 0 && di.GetFiles().Length == 0)
{
FileUtilities.DeleteDirectory(di, true, true);
}
}
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex.Message);
}
if (buildSucceeded)
{
FilingProcessorManager.TheMgr.IncrementCompletedCount();
}
return(buildSucceeded);
}
public void FailsOnMissingOutputDirTest()
{
string ticks = $"{DateTime.Now.Ticks}";
string directoryName = $"dir{ticks}";
string directoryPath = Path.Combine(Path.GetTempPath(), directoryName);
string zipPath = Path.Combine(directoryPath, $"source{ticks}.zip");
string outputDir = Path.Combine(directoryPath, "output");
// Create empty directory
Directory.CreateDirectory(directoryPath);
Assert.IsTrue(Directory.Exists(directoryPath));
// Create file with some data
string fileName = $"file{ticks}.txt";
string filePath = Path.Combine(directoryPath, fileName);
File.WriteAllText(filePath, $"{this.GetType()}");
Assert.IsTrue(File.Exists(filePath));
ZipUtilities.AddFileToArchive(zipPath, filePath);
Assert.IsTrue(File.Exists(zipPath));
// With instance arguments
IAction action = new UnzipArchiveAction()
{
SourceZipPath = zipPath,
};
var actionResult = action.Execute(ArgumentCollection.New());
Assert.IsNotNull(actionResult);
Assert.IsFalse(actionResult.Result);
Assert.IsNotNull(actionResult.AttachedException);
// With output dir defined in execution arguments
((UnzipArchiveAction)action).SourceZipPath = null;
action.Execute(ArgumentCollection.New()
.WithArgument(UnzipArchiveActionExecutionArgs.SourceZipPath, zipPath)
);
Assert.IsNotNull(actionResult);
Assert.IsFalse(actionResult.Result);
Assert.IsNotNull(actionResult.AttachedException);
// Cleanup
if (Directory.Exists(outputDir))
{
Directory.Delete(outputDir, true);
}
if (Directory.Exists(directoryPath))
{
Directory.Delete(directoryPath, true);
}
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(zipPath))
{
File.Delete(zipPath);
}
}
public static void unZipFile(string zipFile, string outputFolder)
{
try
{
if (Directory.Exists(outputFolder))
{
Directory.Delete(outputFolder, true);
}
}
catch { }
ZipUtilities zipUtilities = new ZipUtilities();
zipUtilities.ExtractZipFile(zipFile, outputFolder);
}
/// <summary>
/// Ensure the correct runtime is ready to go. Return a path to the correct folder
/// that contains the APSIM executables.
/// </summary>
/// <param name="environment"></param>
/// <returns></returns>
public static string SetupRunTimeEnvironment(RuntimeEnvironment environment)
{
string binFolder = null;
string environmentFolder = null;
string url = null;
string commandLineArguments = null;
if (environment.APSIMRevision != null)
{
environmentFolder = environment.APSIMRevision;
if (environment.RuntimePackage != null)
{
environmentFolder += "-";
environmentFolder += environment.RuntimePackage;
}
url = @"http://bob.apsim.info/files/" + environment.APSIMRevision + ".binaries.WINDOWS.INTEL.exe";
binFolder = Path.Combine(environmentFolder, "Temp", "Model");
}
if (environment.APSIMxBuildNumber > 0)
{
url = WebUtilities.CallRESTService <string>("http://www.apsim.info/APSIM.Builds.Service/Builds.svc/GetURLOfVersionForIssue?issueid=" +
environment.APSIMxBuildNumber);
environmentFolder = "ApsimX-" + environment.APSIMxBuildNumber;
if (environment.RuntimePackage != null)
{
environmentFolder += "-";
environmentFolder += environment.RuntimePackage;
}
commandLineArguments = "/SILENT /NOICONS /DIR=\".\"";
binFolder = Path.Combine(environmentFolder, "Bin");
}
else if (environment.AusfarmRevision != null)
{
environmentFolder = environment.AusfarmRevision;
if (environment.RuntimePackage != null)
{
environmentFolder += "-";
environmentFolder += environment.RuntimePackage;
}
string packageFileName = Path.Combine("RuntimePackages", environment.AusfarmRevision + ".zip");
ZipUtilities.UnZipFiles(packageFileName, environmentFolder, null);
binFolder = Path.Combine(environmentFolder, "Ausfarm");
}
environmentFolder = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), environmentFolder);
if (!Directory.Exists(environmentFolder))
{
string downloadedFileName = Path.Combine(environmentFolder, "Temp.exe");
// Download the file
DownloadAndExecuteFile(environmentFolder, url, commandLineArguments, downloadedFileName);
// Copy in the extra runtime packages.
if (environment.RuntimePackage != null)
{
string packageFileName = Path.Combine("RuntimePackages", environment.RuntimePackage + ".zip");
ZipUtilities.UnZipFiles(packageFileName, environmentFolder, null);
}
}
return(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), binFolder));
}
public void ZipAndUnzipTest()
{
string sourceFolder = BackupUtilities.CreateTempFilePath();
string destinationFolder = BackupUtilities.CreateTempFilePath();
string zipFile = BackupUtilities.CreateTempFilePath() + ".zip";
try
{
Directory.CreateDirectory(sourceFolder);
string file1Text = "This is test file 1.";
string file2Text = "This is test file 2.";
string file3Text = "This is test file 3.";
string file1 = Path.Combine(sourceFolder, "File1.txt");
string file2Directory = Path.Combine(sourceFolder, "a");
string file2 = Path.Combine(file2Directory, "File2.txt");
string file3Directory = Path.Combine(sourceFolder, "b");
string file3 = Path.Combine(file3Directory, "File3.txt");
File.WriteAllText(file1, file1Text);
Directory.CreateDirectory(file2Directory);
File.WriteAllText(file2, file2Text);
Directory.CreateDirectory(file3Directory);
File.WriteAllText(file3, file3Text);
ZipUtilities.Zip(sourceFolder, zipFile);
Assert.IsTrue(File.Exists(zipFile), "Expected zip file to exist.");
TestUtilities.AssertNotEmpty(zipFile);
ZipUtilities.Unzip(zipFile, destinationFolder);
FileSystemInfo[] fileSystemInfos =
new DirectoryInfo(destinationFolder).GetFileSystemInfos();
Assert.IsTrue(fileSystemInfos.Length > 0, "Unzip did not create any files in the destination directory.");
file1 = Path.Combine(destinationFolder, "File1.txt");
file2Directory = Path.Combine(destinationFolder, "a");
file2 = Path.Combine(file2Directory, "File2.txt");
file3Directory = Path.Combine(destinationFolder, "b");
file3 = Path.Combine(file3Directory, "File3.txt");
Assert.IsTrue(File.Exists(file1), "Unzipped file was not found.");
Assert.AreEqual(file1Text, File.ReadAllText(file1), "Unexpected file contents.");
Assert.IsTrue(File.Exists(file2), "Unzipped file was not found.");
Assert.AreEqual(file2Text, File.ReadAllText(file2), "Unexpected file contents.");
Assert.IsTrue(File.Exists(file3), "Unzipped file was not found.");
Assert.AreEqual(file3Text, File.ReadAllText(file3), "Unexpected file contents.");
}
finally
{
try
{
Directory.Delete(sourceFolder, true);
Directory.Delete(destinationFolder, true);
File.Delete(zipFile);
}
catch
{
}
}
}
