/*
** If it does not already exists, create and populate the on-disk file
** for JournalFile p.
*/
static int createFile(JournalFile p){
int rc = SQLITE_OK;
if( null==p.pReal ){
sqlite3_file pReal = (sqlite3_file )&p[1];
rc = sqlite3OsOpen(p.pVfs, p.zJournal, pReal, p.flags, 0);
if( rc==SQLITE_OK ){
p.pReal = pReal;
if( p.iSize>0 ){
Debug.Assert(p.iSize<=p.nBuf);
rc = sqlite3OsWrite(p.pReal, p.zBuf, p.iSize, 0);
}
}
}
return rc;
}
public void ToStream(StorageEnvironment env, JournalFile journal, long start4Kb,
long numberOf4KbsToCopy, Stream output, Action <string> infoNotify = null, CancellationToken cancellationToken = default)
{
const int pageSize = 4 * Constants.Size.Kilobyte;
var maxNumOf4KbsToCopyAtOnce = _buffer.Length / pageSize;
var page = start4Kb;
var toBeCopied = new Size(numberOf4KbsToCopy * pageSize, SizeUnit.Bytes).ToString();
var totalSw = Stopwatch.StartNew();
var sw = Stopwatch.StartNew();
long totalCopied = 0;
fixed(byte *ptr = _buffer)
{
while (numberOf4KbsToCopy > 0)
{
cancellationToken.ThrowIfCancellationRequested();
var pageCount = Math.Min(maxNumOf4KbsToCopyAtOnce, numberOf4KbsToCopy);
if (journal.JournalWriter.Read(ptr,
pageCount * pageSize,
page * pageSize) == false)
{
throw new InvalidOperationException("Could not read from journal #" + journal.Number + " " +
+pageCount + " pages.");
}
var bytesCount = (int)(pageCount * (4 * Constants.Size.Kilobyte));
output.Write(_buffer, 0, bytesCount);
page += pageCount;
numberOf4KbsToCopy -= pageCount;
totalCopied += bytesCount;
if (sw.ElapsedMilliseconds > 500)
{
infoNotify?.Invoke($"Copied: {new Size(totalCopied, SizeUnit.Bytes)} / {toBeCopied}");
sw.Restart();
}
}
}
var totalSecElapsed = Math.Max((double)totalSw.ElapsedMilliseconds / 1000, 0.0001);
infoNotify?.Invoke($"Finshed copying {new Size(totalCopied, SizeUnit.Bytes)}, " +
$"{new Size((long)(totalCopied / totalSecElapsed), SizeUnit.Bytes)}/sec");
Debug.Assert(numberOf4KbsToCopy == 0);
}
/*
** If it does not already exists, create and populate the on-disk file
** for JournalFile p.
*/
static int createFile(JournalFile p)
{
int rc = SQLITE_OK;
if (null == p.pReal)
{
sqlite3_file pReal = (sqlite3_file) & p[1];
rc = sqlite3OsOpen(p.pVfs, p.zJournal, pReal, p.flags, 0);
if (rc == SQLITE_OK)
{
p.pReal = pReal;
if (p.iSize > 0)
{
Debug.Assert(p.iSize <= p.nBuf);
rc = sqlite3OsWrite(p.pReal, p.zBuf, p.iSize, 0);
}
}
}
return(rc);
}
public void ToStream(JournalFile journal, long startPage, long pagesToCopy, Stream output)
{
var maxNumOfPagesToCopyAtOnce = _buffer.Length / AbstractPager.PageSize;
var page = startPage;
fixed(byte *ptr = _buffer)
{
while (pagesToCopy > 0)
{
var pageCount = Math.Min(maxNumOfPagesToCopyAtOnce, pagesToCopy);
var bytesCount = (int)(pageCount * AbstractPager.PageSize);
journal.JournalWriter.Read(page, ptr, bytesCount);
output.Write(_buffer, 0, bytesCount);
page += pageCount;
pagesToCopy -= pageCount;
}
}
Debug.Assert(pagesToCopy == 0);
}
public long ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
Action <string> infoNotify = null)
{
infoNotify = infoNotify ?? (s => { });
if (env.Options.IncrementalBackupEnabled == false)
{
throw new InvalidOperationException("Incremental backup is disabled for this storage");
}
long numberOfBackedUpPages = 0;
var copier = new DataCopier(AbstractPager.PageSize * 16);
var backupSuccess = true;
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
using (var file = new FileStream(backupPath, FileMode.Create))
using (var package = new ZipArchive(file, ZipArchiveMode.Create))
{
IncrementalBackupInfo backupInfo;
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
backupInfo = env.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition;
}
// txw.Commit(); intentionally not committing
}
using (env.NewTransaction(TransactionFlags.Read))
{
var usedJournals = new List <JournalFile>();
try
{
long lastBackedUpPage = -1;
long lastBackedUpFile = -1;
var firstJournalToBackup = backupInfo.LastBackedUpJournal;
if (firstJournalToBackup == -1)
{
firstJournalToBackup = 0; // first time that we do incremental backup
}
for (var journalNum = firstJournalToBackup; journalNum <= backupInfo.LastCreatedJournal; journalNum++)
{
var num = journalNum;
var journalFile = env.Journal.Files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
}
journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
var startBackupAt = 0L;
var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
if (journalFile.Number == backupInfo.LastBackedUpJournal)
{
startBackupAt = backupInfo.LastBackedUpJournalPage + 1;
pagesToCopy -= startBackupAt;
}
if (startBackupAt >= journalFile.JournalWriter.NumberOfAllocatedPages) // nothing to do here
{
continue;
}
var part = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalNum), compression);
Debug.Assert(part != null);
if (journalFile.Number == lastWrittenLogFile)
{
pagesToCopy -= (journalFile.JournalWriter.NumberOfAllocatedPages - lastWrittenLogPage);
}
using (var stream = part.Open())
{
copier.ToStream(journalFile, startBackupAt, pagesToCopy, stream);
infoNotify(string.Format("Voron Incr copy journal number {0}", num));
}
lastBackedUpFile = journalFile.Number;
if (journalFile.Number == backupInfo.LastCreatedJournal)
{
lastBackedUpPage = startBackupAt + pagesToCopy - 1;
// we used all of this file, so the next backup should start in the next file
if (lastBackedUpPage == (journalFile.JournalWriter.NumberOfAllocatedPages - 1))
{
lastBackedUpPage = -1;
lastBackedUpFile++;
}
}
numberOfBackedUpPages += pagesToCopy;
}
//Debug.Assert(lastBackedUpPage != -1);
env.HeaderAccessor.Modify(header =>
{
header->IncrementalBackup.LastBackedUpJournal = lastBackedUpFile;
header->IncrementalBackup.LastBackedUpJournalPage = lastBackedUpPage;
});
}
catch (Exception)
{
backupSuccess = false;
throw;
}
finally
{
foreach (var jrnl in usedJournals)
{
if (backupSuccess) // if backup succeeded we can remove journals
{
if (jrnl.Number < lastWrittenLogFile) // prevent deletion of the current journal and journals with a greater number
{
jrnl.DeleteOnClose = true;
}
}
jrnl.Release();
}
}
infoNotify(string.Format("Voron Incr Backup total {0} pages", numberOfBackedUpPages));
return(numberOfBackedUpPages);
}
}
}
private static void Backup(
StorageEnvironment env, CompressionLevel compression, Action <string> infoNotify,
Action backupStarted, AbstractPager dataPager, ZipArchive package, string basePath, DataCopier copier)
{
var usedJournals = new List <JournalFile>();
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
LowLevelTransaction txr = null;
try
{
long allocatedPages;
var writePesistentContext = new TransactionPersistentContext(true);
var readPesistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(writePesistentContext, TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
{
txr = env.NewLowLevelTransaction(readPesistentContext, TransactionFlags.Read); // now have snapshot view
allocatedPages = dataPager.NumberOfAllocatedPages;
Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
infoNotify("Voron copy headers for " + basePath);
VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options, basePath);
// journal files snapshot
var files = env.Journal.Files; // thread safety copy
JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1;
journalNum <= journalInfo.CurrentJournal;
journalNum++)
{
var journalFile = files.FirstOrDefault(x => x.Number == journalNum);
// first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Bits.NextPowerOf2(pager.NumberOfAllocatedPages * env.Options.PageSize);
}
journalFile = new JournalFile(env, env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
}
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
}
// txw.Commit(); intentionally not committing
}
backupStarted?.Invoke();
// data file backup
var dataPart = package.CreateEntry(Path.Combine(basePath, Constants.DatabaseFilename), compression);
Debug.Assert(dataPart != null);
if (allocatedPages > 0) //only true if dataPager is still empty at backup start
{
using (var dataStream = dataPart.Open())
{
// now can copy everything else
copier.ToStream(dataPager, 0, allocatedPages, dataStream);
}
}
try
{
foreach (JournalFile journalFile in usedJournals)
{
var entryName = Path.Combine(basePath, StorageEnvironmentOptions.JournalName(journalFile.Number));
var journalPart = package.CreateEntry(entryName, compression);
Debug.Assert(journalPart != null);
long pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
if (journalFile.Number == lastWrittenLogFile)
{
pagesToCopy = lastWrittenLogPage + 1;
}
using (var stream = journalPart.Open())
{
copier.ToStream(env, journalFile, 0, pagesToCopy, stream);
infoNotify(string.Format("Voron copy journal file {0}", entryName));
}
}
}
finally
{
foreach (var journalFile in usedJournals)
{
journalFile.Release();
}
}
}
finally
{
txr?.Dispose();
}
}
/// <summary>
/// Constructs the entry.
/// </summary>
/// <param name="resource_name"></param>
/// <param name="journal"></param>
/// <param name="position"></param>
/// <param name="page_number"></param>
public JournalEntry(String resource_name, JournalFile journal, long position, long page_number)
{
this.resource_name = resource_name;
this.journal = journal;
this.position = position;
this.page_number = page_number;
}
/// <summary>
/// Recovers any lost operations that are currently input the journal.
/// </summary>
/// <remarks>
/// This retries all logged entries. This would typically be called before
/// any other IO operations.
/// </remarks>
private void RollForwardRecover()
{
// Console.Out.WriteLine("RollForwardRecover()");
// The list of all journal files,
ArrayList journal_files_list = new ArrayList();
// Scan the journal path for any journal files.
for (int i = 10; i < 74; ++i) {
String journal_fn = GetJournalFileName(i);
string f = Path.Combine(journal_path, journal_fn);
// If the journal exists, create a summary of the journal
if (File.Exists(f)) {
if (read_only) {
throw new IOException(
"Journal file " + f + " exists for a Read-only session. " +
"There may not be any pending journals for a Read-only session.");
}
JournalFile jf = new JournalFile(this, f, read_only);
// Open the journal file for recovery. This will set various
// information about the journal such as the last check point and the
// id of the journal file.
JournalSummary summary = jf.OpenForRecovery();
// If the journal can be recovered from.
if (summary.can_be_recovered) {
if (Debug.IsInterestedIn(LogLevel.Information)) {
Debug.Log(LogLevel.Information, this, "Journal " + jf +
" found - can be recovered.");
}
journal_files_list.Add(summary);
} else {
if (Debug.IsInterestedIn(LogLevel.Information)) {
Debug.Log(LogLevel.Information, this, "Journal " + jf +
" deleting - nothing to recover.");
}
// Otherwise close and delete it
jf.CloseAndDelete();
}
}
}
// if (journal_files_list.size() == 0) {
// Console.Out.WriteLine("Nothing to recover.");
// }
// Sort the journal file list from oldest to newest. The oldest journals
// are recovered first.
journal_files_list.Sort(journal_list_comparator);
long last_journal_number = -1;
// Persist the journals
for (int i = 0; i < journal_files_list.Count; ++i) {
JournalSummary summary = (JournalSummary)journal_files_list[i];
// Check the resources for this summary
ArrayList res_list = summary.resource_list;
for (int n = 0; n < res_list.Count; ++n) {
String resource_name = (String)res_list[n];
// This puts the resource into the hash map.
IJournalledResource resource = CreateResource(resource_name);
}
// Assert that we are recovering the journals input the correct order
JournalFile jf = summary.journal_file;
if (jf.journal_number < last_journal_number) {
throw new ApplicationException("Assertion failed, sort failed.");
}
last_journal_number = jf.journal_number;
if (Debug.IsInterestedIn(LogLevel.Information)) {
Debug.Log(LogLevel.Information, this, "Recovering: " + jf +
" (8 .. " + summary.last_checkpoint + ")");
}
jf.Persist(8, summary.last_checkpoint);
// Then close and delete.
jf.CloseAndDelete();
// Check the resources for this summary and close them
for (int n = 0; n < res_list.Count; ++n) {
String resource_name = (String)res_list[n];
AbstractResource resource =
(AbstractResource)CreateResource(resource_name);
// When we finished, make sure the resource is closed again
// Close the resource
resource.PersistClose();
// Post recover notification
resource.OnPostRecover();
}
}
}
/// <summary>
/// Creates a new top journal file.
/// </summary>
private void NewTopJournalFile()
{
// // Move the old journal to the archive?
// if (top_journal_file != null) {
// journal_archives.add(top_journal_file);
// }
String journal_fn = GetJournalFileName((int)((journal_number & 63) + 10));
++journal_number;
string f = Path.Combine(journal_path, journal_fn);
if (File.Exists(f)) {
throw new IOException("Journal file already exists.");
}
top_journal_file = new JournalFile(this, f, read_only);
top_journal_file.Open(journal_number - 1);
}
public JournalSummary(JournalFile journal_file)
{
this.journal_file = journal_file;
}
private static void Backup(
StorageEnvironment env, CompressionLevel compression, Action <string> infoNotify,
Action backupStarted, AbstractPager dataPager, ZipArchive package, string basePath, DataCopier copier)
{
var usedJournals = new List <JournalFile>();
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
LowLevelTransaction txr = null;
var backupSuccess = false;
try
{
long allocatedPages;
var writePesistentContext = new TransactionPersistentContext(true);
var readPesistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(writePesistentContext, TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
{
txr = env.NewLowLevelTransaction(readPesistentContext, TransactionFlags.Read); // now have snapshot view
allocatedPages = dataPager.NumberOfAllocatedPages;
Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
infoNotify("Voron copy headers for " + basePath);
VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options, basePath);
// journal files snapshot
var files = env.Journal.Files; // thread safety copy
JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1;
journalNum <= journalInfo.CurrentJournal;
journalNum++)
{
var journalFile = files.FirstOrDefault(x => x.Number == journalNum);
// first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Bits.NextPowerOf2(pager.NumberOfAllocatedPages * Constants.Storage.PageSize);
}
journalFile = new JournalFile(env, env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
}
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePosIn4KbPosition - 1;
}
// txw.Commit(); intentionally not committing
}
backupStarted?.Invoke();
// data file backup
var dataPart = package.CreateEntry(Path.Combine(basePath, Constants.DatabaseFilename), compression);
Debug.Assert(dataPart != null);
if (allocatedPages > 0) //only true if dataPager is still empty at backup start
{
using (var dataStream = dataPart.Open())
{
// now can copy everything else
copier.ToStream(dataPager, 0, allocatedPages, dataStream);
}
}
try
{
long lastBackedupJournal = 0;
foreach (var journalFile in usedJournals)
{
var entryName = StorageEnvironmentOptions.JournalName(journalFile.Number);
var journalPart = package.CreateEntry(Path.Combine(basePath, entryName), compression);
Debug.Assert(journalPart != null);
long pagesToCopy = journalFile.JournalWriter.NumberOfAllocated4Kb;
if (journalFile.Number == lastWrittenLogFile)
{
pagesToCopy = lastWrittenLogPage + 1;
}
using (var stream = journalPart.Open())
{
copier.ToStream(env, journalFile, 0, pagesToCopy, stream);
infoNotify(string.Format("Voron copy journal file {0}", entryName));
}
lastBackedupJournal = journalFile.Number;
}
if (env.Options.IncrementalBackupEnabled)
{
env.HeaderAccessor.Modify(header =>
{
header->IncrementalBackup.LastBackedUpJournal = lastBackedupJournal;
//since we backed-up everything, no need to start next incremental backup from the middle
header->IncrementalBackup.LastBackedUpJournalPage = -1;
});
}
backupSuccess = true;
}
catch (Exception)
{
backupSuccess = false;
throw;
}
finally
{
var lastSyncedJournal = env.HeaderAccessor.Get(header => header->Journal).LastSyncedJournal;
foreach (var journalFile in usedJournals)
{
if (backupSuccess) // if backup succeeded we can remove journals
{
if (journalFile.Number < lastWrittenLogFile && // prevent deletion of the current journal and journals with a greater number
journalFile.Number < lastSyncedJournal) // prevent deletion of journals that aren't synced with the data file
{
journalFile.DeleteOnClose = true;
}
}
journalFile.Release();
}
}
}
finally
{
txr?.Dispose();
}
}
public static void Main(string[] args)
{
bool showHelp = false;
bool? extractUnknowns = null;
bool overwriteFiles = false;
bool verbose = true;
string pattern = null;
var options = new OptionSet()
{
{ "o|overwrite", "overwrite existing files", v => overwriteFiles = v != null },
{
"nu|no-unknowns", "don't extract unknown files",
v => extractUnknowns = v != null ? false : extractUnknowns
},
{
"ou|only-unknowns", "only extract unknown files",
v => extractUnknowns = v != null ? true : extractUnknowns
},
{ "v|verbose", "be verbose", v => verbose = v != null },
{ "f|filter=", "match files using pattern", v => pattern = v },
{ "h|help", "show this message and exit", v => showHelp = v != null },
};
List <string> extras;
try
{
extras = options.Parse(args);
}
catch (OptionException e)
{
Console.Write("{0}: ", GetExecutableName());
Console.WriteLine(e.Message);
Console.WriteLine("Try `{0} --help' for more information.", GetExecutableName());
return;
}
if (extras.Count < 1 ||
extras.Count > 2 ||
showHelp == true)
{
Console.WriteLine("Usage: {0} [OPTIONS]+ input_file.hogg [output_dir]", GetExecutableName());
Console.WriteLine();
Console.WriteLine("Options:");
options.WriteOptionDescriptions(Console.Out);
return;
}
string inputPath = extras[0];
string outputPath = extras.Count > 1 ? extras[1] : Path.ChangeExtension(inputPath, null) + "_unpack";
Regex regex = null;
if (string.IsNullOrEmpty(pattern) == false)
{
regex = new Regex(pattern, RegexOptions.Compiled | RegexOptions.IgnoreCase);
}
using (var input = File.OpenRead(inputPath))
{
var hog = new HogFile();
hog.Deserialize(input);
var dataList = new Dictionary <int, byte[]>();
var dataListEntry = hog.Files
.SingleOrDefault(e => e.AttributeId == 0 && e.Size != -1);
if (dataListEntry != null)
{
using (var data = ReadFileData(hog, dataListEntry, input))
{
if (data.ReadValueU32(hog.Endian) != 0)
{
throw new FormatException();
}
var count = data.ReadValueS32(hog.Endian);
if (count < 0)
{
throw new FormatException();
}
for (int i = 0; i < count; i++)
{
var length = data.ReadValueU32(hog.Endian);
dataList.Add(i, data.ReadBytes((int)length));
}
}
}
using (var data = new MemoryStream(hog.DataListJournal))
{
var journal = new JournalFile()
{
Endian = hog.Endian,
};
journal.Deserialize(data);
foreach (var entry in journal.Entries)
{
if (entry.Action == Journal.Action.Add)
{
dataList[entry.TargetId] = (byte[])entry.Data.Clone();
}
else if (entry.Action == Journal.Action.Remove)
{
if (dataList.Remove(entry.TargetId) == false)
{
throw new InvalidOperationException();
}
}
else
{
throw new InvalidOperationException();
}
}
}
var consumedAttributes = new List <int>();
var files = hog.Files
.Where(e => e.Size != -1 && e.AttributeId != 0)
.ToArray();
long current = 0;
long total = files.Count();
var names = new Dictionary <int, string>();
foreach (var entry in files)
{
current++;
if (entry.Unknown5 != -2 || entry.AttributeId == -1)
{
throw new InvalidOperationException();
}
if (entry.AttributeId < 0 || entry.AttributeId >= hog.Attributes.Count)
{
throw new InvalidOperationException();
}
if ((hog.Attributes[entry.AttributeId].Flags & 1) == 1) // entry is unused
{
throw new InvalidOperationException();
}
if (consumedAttributes.Contains(entry.AttributeId) == true)
{
throw new InvalidOperationException();
}
consumedAttributes.Add(entry.AttributeId);
var attribute = hog.Attributes[entry.AttributeId];
string name;
if (dataList.ContainsKey(attribute.NameId) == false)
{
name = Path.Combine("__UNKNOWN", attribute.NameId.ToString(CultureInfo.InvariantCulture));
}
else
{
if (names.ContainsKey(attribute.NameId) == true)
{
name = names[attribute.NameId];
}
else
{
using (var temp = new MemoryStream(dataList[attribute.NameId]))
{
name = temp.ReadStringZ(Encoding.UTF8);
names.Add(attribute.NameId, name);
}
}
name = name.Replace('/', Path.DirectorySeparatorChar);
if (name.StartsWith(Path.DirectorySeparatorChar.ToString(CultureInfo.InvariantCulture)) == true)
{
name = name.Substring(1);
}
}
if (regex != null && regex.IsMatch(name) == false)
{
continue;
}
var entryPath = Path.Combine(outputPath, name);
if (overwriteFiles == false &&
File.Exists(entryPath) == true)
{
continue;
}
var entryDirectory = Path.GetDirectoryName(entryPath);
if (entryDirectory != null)
{
Directory.CreateDirectory(entryDirectory);
}
if (verbose == true)
{
Console.WriteLine("[{0}/{1}] {2}", current, total, name);
}
using (var output = File.Create(entryPath))
{
ReadFileData(hog, entry, input, output);
}
}
}
}
public void ToFile(StorageEnvironment env, string backupPath, CompressionLevel compression = CompressionLevel.Optimal,
Action <string> infoNotify = null)
{
infoNotify = infoNotify ?? (s => { });
var dataPager = env.Options.DataPager;
var copier = new DataCopier(AbstractPager.PageSize * 16);
Transaction txr = null;
try
{
infoNotify("Voron copy headers");
using (var file = new FileStream(backupPath, FileMode.Create))
using (var package = new ZipArchive(file, ZipArchiveMode.Create))
{
long allocatedPages;
ImmutableAppendOnlyList <JournalFile> files; // thread safety copy
var usedJournals = new List <JournalFile>();
long lastWrittenLogPage = -1;
long lastWrittenLogFile = -1;
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite)) // so we can snapshot the headers safely
{
txr = env.NewTransaction(TransactionFlags.Read); // now have snapshot view
allocatedPages = dataPager.NumberOfAllocatedPages;
Debug.Assert(HeaderAccessor.HeaderFileNames.Length == 2);
VoronBackupUtil.CopyHeaders(compression, package, copier, env.Options);
// journal files snapshot
files = env.Journal.Files;
JournalInfo journalInfo = env.HeaderAccessor.Get(ptr => ptr->Journal);
for (var journalNum = journalInfo.CurrentJournal - journalInfo.JournalFilesCount + 1; journalNum <= journalInfo.CurrentJournal; journalNum++)
{
var journalFile = files.FirstOrDefault(x => x.Number == journalNum); // first check journal files currently being in use
if (journalFile == null)
{
long journalSize;
using (var pager = env.Options.OpenJournalPager(journalNum))
{
journalSize = Utils.NearestPowerOfTwo(pager.NumberOfAllocatedPages * AbstractPager.PageSize);
}
journalFile = new JournalFile(env.Options.CreateJournalWriter(journalNum, journalSize), journalNum);
}
journalFile.AddRef();
usedJournals.Add(journalFile);
}
if (env.Journal.CurrentFile != null)
{
lastWrittenLogFile = env.Journal.CurrentFile.Number;
lastWrittenLogPage = env.Journal.CurrentFile.WritePagePosition - 1;
}
// txw.Commit(); intentionally not committing
}
// data file backup
var dataPart = package.CreateEntry(Constants.DatabaseFilename, compression);
Debug.Assert(dataPart != null);
if (allocatedPages > 0) //only true if dataPager is still empty at backup start
{
using (var dataStream = dataPart.Open())
{
// now can copy everything else
var firstDataPage = dataPager.Read(0);
copier.ToStream(firstDataPage.Base, AbstractPager.PageSize * allocatedPages, dataStream);
}
}
try
{
foreach (var journalFile in usedJournals)
{
var journalPart = package.CreateEntry(StorageEnvironmentOptions.JournalName(journalFile.Number), compression);
Debug.Assert(journalPart != null);
var pagesToCopy = journalFile.JournalWriter.NumberOfAllocatedPages;
if (journalFile.Number == lastWrittenLogFile)
{
pagesToCopy = lastWrittenLogPage + 1;
}
using (var stream = journalPart.Open())
{
copier.ToStream(journalFile, 0, pagesToCopy, stream);
infoNotify(string.Format("Voron copy journal file {0} ", journalFile));
}
}
}
finally
{
foreach (var journalFile in usedJournals)
{
journalFile.Release();
}
}
}
}
finally
{
if (txr != null)
{
txr.Dispose();
}
}
infoNotify(string.Format("Voron backup db finished"));
}
