public static DiscFileSystem FormatFileSystemWithTemplate(Stream template, Stream output,
long firstSector, long sectorCount, long availableSpace)
{
if (NtfsFileSystem.Detect(template))
{
var templateFS = new NtfsFileSystem(template);
var newSize = Math.Min(templateFS.Size, availableSpace);
Logger.Verbose("Formatting stream as <NTFS> file system ({0}B, 0x{1}-0x{2})",
newSize, firstSector.ToString("X2"), sectorCount.ToString("X2"));
return(NtfsFileSystem.Format(output, templateFS.VolumeLabel,
Geometry.FromCapacity(newSize), firstSector, sectorCount,
templateFS.ReadBootCode()));
}
/* else if (FatFileSystem.Detect(template))
* {
* var templateFS = new FatFileSystem(template);
* var newSize = Math.Min(templateFS.Size, availableSpace);
*
* Logger.Verbose("Formatting streams as <FAT> ({0}B, 0x{1}-0x{2})",
* newSize, firstSector.ToString("X2"), sectorCount.ToString("X2"));
*
* return FatFileSystem.FormatPartition(output, templateFS.VolumeLabel,
* Geometry.FromCapacity(newSize), (int)firstSector, (int)sectorCount, 13);
* } */
return(null);
}
public override DiscUtils.FileSystemInfo[] Detect(Stream stream, VolumeInfo volume)
{
if (NtfsFileSystem.Detect(stream))
{
return(new DiscUtils.FileSystemInfo[] { new VfsFileSystemInfo("NTFS", "Microsoft NTFS", Open) });
}
return(new DiscUtils.FileSystemInfo[0]);
}
public static DiscFileSystem GetFileSystem(Stream stream)
{
if (NtfsFileSystem.Detect(stream))
{
Logger.Verbose("Loading stream as <NTFS> file system");
return(new NtfsFileSystem(stream));
}
/* if (FatFileSystem.Detect(stream))
* {
* Logger.Verbose("Loading stream as <FAT> file system");
* return new FatFileSystem(stream);
* } */
return(null);
}
public static bool IsStreamSupportedFileSystem(Stream stream)
{
if (NtfsFileSystem.Detect(stream))
{
Logger.Verbose("Detected stream as <NTFS> file system");
return(true);
}
/* if (FatFileSystem.Detect(stream))
* {
* Logger.Verbose("Detected stream as <FAT> file-system");
* return true;
* } */
// only NTFS can be detected and formatted right now
return(false);
}
private static DiscFileSystem DetectFileSystem(PartitionInfo Partition)
{
using (var stream = Partition.Open())
{
if (NtfsFileSystem.Detect(stream))
{
return(new NtfsFileSystem(Partition.Open()));
}
stream.Seek(0, SeekOrigin.Begin);
if (FatFileSystem.Detect(stream))
{
return(new FatFileSystem(Partition.Open()));
}
/* Ext2/3/4 file system - when Ext becomes fully writable
*
* stream.Seek(0, SeekOrigin.Begin);
* if (ExtFileSystem.Detect(stream))
* return new ExtFileSystem(Partition.Open());
*/
return(null);
}
}
private static int StartInternal()
{
if (!GuidPartitionTable.Detect(mSourceStream))
{
throw new InvalidDataException(
"Source does not contain a supported partitioning scheme");
}
Logger.Debug("Detecting disc geometry of source...");
var geometry = GuidPartitionTable.DetectGeometry(mSourceStream);
Logger.Debug("Reading GPT partition table from source...");
var partitionTable = new GuidPartitionTable(mSourceStream, geometry);
if (partitionTable.Count <= 0)
{
return(SUCCESS); // nothing to clone here
}
// adjust source length
if (FixedLengthStream.IsFixedDiskStream(mSourceStream))
{
mSourceStream.SetLength(geometry.Capacity);
}
Geometry destGeometry = null;
if (FixedLengthStream.IsFixedDiskStream(mDestinationStream))
{
// If we write to a disk, we need our own geometry for that destination
destGeometry = Geometry.FromCapacity(mDestinationStream.Length,
geometry.BytesPerSector);
}
else
{
// If we are just writing to a file, we take the most exact copy we get
destGeometry = geometry;
}
// Set the new size of the destination stream
Logger.Verbose("Updating length of destination: {0} -> {1}",
mDestinationStream.Length, destGeometry.Capacity);
mDestinationStream.SetLength(destGeometry.Capacity);
if (mFullClone)
{
Logger.Warn("Starting full clone...");
CloneStream(0, 0, mSourceStream, mDestinationStream);
return(SUCCESS);
}
Logger.Debug("Initializing new GPT partition table on destination...");
var destPartitionTable = GuidPartitionTable.Initialize(mDestinationStream,
destGeometry);
var availableSpace = destGeometry.Capacity;
var usedSpace = 0L;
// correct available space by remove unusable sectors
// heading sectors
availableSpace -= (destPartitionTable.FirstUsableSector
* destGeometry.BytesPerSector);
// ending sectors
availableSpace -= ((destGeometry.TotalSectorsLong
- destPartitionTable.FirstUsableSector
- destPartitionTable.LastUsableSector)
* destGeometry.BytesPerSector);
// 1. calculate unresizeable space if we are required too
if (destGeometry.Capacity < geometry.Capacity)
{
Logger.Info("Calculating space for new destination partitions...");
var unresizeableSpace = 0L;
for (int i = 0; i < partitionTable.Count; i++)
{
var srcPartition = (GuidPartitionInfo)partitionTable[i];
var srcPartitionStream = srcPartition.Open();
var srcPartitionType = GetWellKnownPartitionType(srcPartition);
var srcPartitionSize = srcPartition.SectorCount *
geometry.BytesPerSector;
var isResizeable = true;
// If the file system is not support, we have to perform
// a byte-for-byte cloning and cannot resize this partition
if (!DiscHandler.IsStreamSupportedFileSystem(srcPartitionStream))
{
Logger.Verbose("Partition{0}: contains no or unsupported file system");
isResizeable = false;
}
// If we have a critical partition, we shouldn't resize that either
if (!IsSupportedPartitionType(srcPartitionType))
{
Logger.Verbose("Partition{0}: unsupported partition type");
isResizeable = false;
}
// if caller want to do a byte-for-byte clone, mark every partition
// as unresizeable
if (mForceExactCloning)
{
isResizeable = false;
}
Logger.Debug("Partition #{0} on source: isResizeable={1}", i,
isResizeable);
if (!isResizeable)
{
// if it's not resizeable, account the space and continue
unresizeableSpace += srcPartitionSize;
Logger.Debug("Partition #{0} on source: unresizeable, size is {1} " +
"(total unresizable {2}/used {3})", i,
srcPartitionSize, unresizeableSpace, usedSpace);
}
else
{
// if it is resizeable, we have to report how much space we need
var srcPartitionFS = DiscHandler.GetFileSystem(srcPartitionStream);
usedSpace += srcPartitionFS.UsedSpace;
Logger.Debug("Partition #{0} on source: resizeable, space is {1} " +
"(total unresizable {2}/used {3})", i,
usedSpace, unresizeableSpace, usedSpace);
}
}
// reduce the dynamically available space ...
availableSpace -= unresizeableSpace;
// ... and assert it
if (availableSpace < 0)
{
throw new InvalidOperationException("Cannot clone, destination is " +
String.Format("{0:#,##0}", -availableSpace) + " Bytes too small");
}
// assert the used space too
if (availableSpace - usedSpace < 0)
{
throw new InvalidOperationException("Cannot clone, destination is " +
String.Format("{0:#,##0}", availableSpace - usedSpace) +
" Bytes too small");
}
}
else
{
Logger.Info("Destination can contain source, no need to resize partitions!");
}
// 2. calculate the size for each new partition
var destPartitionSizes = new long[partitionTable.Count];
for (int i = 0; i < partitionTable.Count; i++)
{
var srcPartition = (GuidPartitionInfo)partitionTable[i];
var srcPartitionStream = srcPartition.Open();
var srcPartitionType = GetWellKnownPartitionType(srcPartition);
var srcPartitionSize = srcPartition.SectorCount *
geometry.BytesPerSector;
// if the destination can take more data, skip every check
if (geometry.Capacity <= destGeometry.Capacity)
{
Logger.Debug("Partition{0}: Destination can contain full partition, continue...", i);
destPartitionSizes[i] = srcPartitionSize;
availableSpace -= srcPartitionSize;
continue;
}
var isResizeable = true;
// If the device-systme is not support, we have to perform
// a byte-for-byte cloning and cannot resize this partition
if (!DiscHandler.IsStreamSupportedFileSystem(srcPartitionStream))
{
isResizeable = false;
}
// If we have a critical partition, we shouldn't resize that either
if (!IsSupportedPartitionType(srcPartitionType))
{
isResizeable = false;
}
// if caller want to do a byte-for-byte clone, mark every partition
// as unresizeable
if (mForceExactCloning)
{
isResizeable = false;
}
Logger.Debug("Partition #{0} on source: isResizeable={1}", i,
isResizeable);
// If our friend is not resizeable, set size and stop processing it
if (!isResizeable)
{
destPartitionSizes[i] = srcPartitionSize;
Logger.Debug("Partition #{0} on source: unresizeable, size is {1} " +
"(total available {2}/used {3})", i,
usedSpace, availableSpace, usedSpace);
// DO NOT ALIGN <availableSpace> HERE!!!
// Has already been done in step 1
continue;
}
//
// OK. If we are here, a resizeable partition awaits processing
//
var srcPartitionFS = DiscHandler.GetFileSystem(srcPartitionStream);
// First we need to check if we need to reqsize the current partition.
// For that, calculate the space that is left for future partitions. If the
// result is OK (less or equal to the available size), we can skip
// resizing the the current one.
// Make sure to remove the factor of this partition from usedSpace first
if (((usedSpace - srcPartitionFS.UsedSpace) + srcPartitionSize) <= availableSpace)
{
// update usedSpace
usedSpace -= srcPartitionFS.UsedSpace;
// align availableSpace
availableSpace -= srcPartitionSize;
// we are good to skip resizing this one. assign size and early exit
destPartitionSizes[i] = srcPartitionSize;
Logger.Debug(
"Partition #{0} on source: resizeable, space is {1}, size is {2} " +
"(total available {3}/used {4})", i,
srcPartitionFS.UsedSpace, srcPartitionSize, availableSpace, usedSpace);
continue;
}
// So this partition is too large, let's resize it to the biggest size possible
var maxPartitionSize = Math.Max(
// Occupied space is the absolute minimal space we need
srcPartitionFS.UsedSpace,
// This is the largest space possible. Take the still available space
// and remove still required space, while also remove the factor for
// the current partition
availableSpace - (usedSpace - srcPartitionFS.UsedSpace)
);
Logger.Debug(
"Partition #{0} on source: resizeable, max. space is {1} " +
"(total available {2}/used {3})", i,
maxPartitionSize, availableSpace, usedSpace);
// update usedSpace
usedSpace -= srcPartitionFS.UsedSpace;
// align availableSpace
availableSpace -= maxPartitionSize;
destPartitionSizes[i] = maxPartitionSize;
}
// a last assert of the available space, just to be sure
if (availableSpace < 0)
{
throw new InvalidOperationException("Cannot clone, destination is " +
String.Format("{0:#.##0}", -availableSpace) + " Bytes too small");
}
// 2. create the new partitions with the aligned sizes
for (int i = 0; i < partitionTable.Count; i++)
{
var srcPartition = (GuidPartitionInfo)partitionTable[i];
var srcPartitionStream = srcPartition.Open();
var srcPartitionType = GetWellKnownPartitionType(srcPartition);
// manueal type adjusting
if (NtfsFileSystem.Detect(srcPartitionStream))
{
srcPartitionType = WellKnownPartitionType.WindowsNtfs;
}
else if (FatFileSystem.Detect(srcPartitionStream))
{
srcPartitionType = WellKnownPartitionType.WindowsFat;
}
var destPartitionSize = destPartitionSizes[i];
Logger.Debug("Creating partition table: #{0}; {1}/{2}@0x{3}-{4}", i,
srcPartition.Name, srcPartition.Identity,
srcPartition.FirstSector.ToString("X2"), destPartitionSize);
destPartitionTable.Create(
destPartitionSize,
srcPartitionType,
true //doesn't matter on GPT tables
);
}
// 3. make sure the count of partitions is the same
if (partitionTable.Count != destPartitionTable.Count)
{
throw new InvalidOperationException(
"Failed to create proper GUID partition table");
}
// 4. do the real cloning
for (int i = 0; i < destPartitionTable.Count; i++)
{
var srcPartition = (GuidPartitionInfo)partitionTable[i];
var srcPartitionStream = srcPartition.Open();
var srcPartitionType = GetWellKnownPartitionType(srcPartition);
var destPartition = (GuidPartitionInfo)destPartitionTable[i];
var destPartitionStream = destPartition.Open();
var destPartitionSize = destPartitionSizes[i];
var requiresExactCloning = false;
// To support all possible file-systems, perform a byte-for-byte
// cloning if the file-system is not supported by us.
if (!DiscHandler.IsStreamSupportedFileSystem(srcPartitionStream))
{
requiresExactCloning = true;
}
// If we have a critical partition, we should skip that one too
if (srcPartitionType == WellKnownPartitionType.BiosBoot ||
srcPartitionType == WellKnownPartitionType.EfiSystem ||
srcPartitionType == WellKnownPartitionType.MicrosoftReserved)
{
requiresExactCloning = true;
}
// if caller want to do a byte-for-byte clone, let's enable it
if (mForceExactCloning)
{
requiresExactCloning = true;
}
if (requiresExactCloning)
{
var taskId = (mTaskIdCounter++);
Logger.Info("[{0}/{1}] cp-fs : {2}/{3}@0x{4}", i, taskId,
srcPartition.Name, srcPartition.Identity,
srcPartition.FirstSector.ToString("X2"));
CloneStream(i, taskId, srcPartitionStream, destPartitionStream);
}
else
{
var srcPartitionFS = DiscHandler.GetFileSystem(srcPartitionStream);
var srcPartitionBootCode = srcPartitionFS.ReadBootCode();
var destPartitionFS = DiscHandler.FormatFileSystemWithTemplate(
srcPartitionStream, destPartitionStream, destPartition.FirstSector,
destPartition.SectorCount, destPartitionSize
);
// Tracks all NTFS file IDs for hard link recognition
// <Source FS File ID, Destination FS File ID>
var ntfsHardlinkTracker = new Dictionary <uint, uint>();
// last clone each single with here
Action <DiscDirectoryInfo, DiscDirectoryInfo> cloneSrcFsToDestFs = null;
cloneSrcFsToDestFs = new Action <DiscDirectoryInfo, DiscDirectoryInfo>(
(sourceDir, destDir) =>
{
// recursive enumeration. save to create directory without checks for
// parent directories.
var taskId = (mTaskIdCounter++);
Logger.Info("[{0}/{1}] mk-dir : {2}", i, taskId, destDir.FullName);
destDir.Create();
// copy files if there are any
foreach (var sourceFile in sourceDir.GetFiles())
{
var skipCopying = false;
taskId = (mTaskIdCounter++);
var sourceFileStream = sourceFile.Open(FileMode.Open,
FileAccess.Read);
var destFileName = Path.Combine(destDir.FullName, sourceFile.Name);
Logger.Info("[{0}/{1}] mk-file: {2}", i, taskId,
destFileName);
var destFileStream = destPartitionFS.OpenFile(
destFileName,
FileMode.Create, FileAccess.Write);
var destFile = new DiscFileInfo(destPartitionFS, destFileName);
// NTFS hard link handling
if (destPartitionFS is NtfsFileSystem)
{
var ntfsSourceFS = (NtfsFileSystem)srcPartitionFS;
var ntfsDestFS = (NtfsFileSystem)destPartitionFS;
var sourceFileNtfsEntry = ntfsSourceFS.GetDirectoryEntry(
sourceFile.FullName);
var sourceFileNtfsRef = ntfsSourceFS.GetFile(
sourceFileNtfsEntry.Reference);
var destFileNtfsEntry = ntfsDestFS.GetDirectoryEntry(
destFile.FullName);
var destFileNtfsRef = ntfsDestFS.GetFile(
destFileNtfsEntry.Reference);
var sourceFileNtfsId = sourceFileNtfsRef.IndexInMft;
// check if this files was already processed once
if (ntfsHardlinkTracker.ContainsKey(sourceFileNtfsId))
{
var trackedDestFileNtfsRef = ntfsDestFS.GetFile(
ntfsHardlinkTracker[sourceFileNtfsId]);
// if we have a hardlink-match, close the old stream
// and delete the file
destFileStream.Close();
destFile.Delete();
// then create the hardlink and mention that we don't
// want/need to copy the content anymore
Logger.Info("[{0}/{1}] mk-lnk : {2} => {3}", i, taskId,
sourceFileNtfsRef.Names[0], destFile.FullName);
ntfsDestFS.CreateHardLink(
trackedDestFileNtfsRef.DirectoryEntry,
destFile.FullName);
skipCopying = true;
}
else
{
Logger.Verbose("[{0}/{1}] rg-lnk : {2}#{3} -> {4}#{5}", i, taskId,
sourceFileNtfsRef.Names[0], sourceFileNtfsRef.IndexInMft,
destFileNtfsRef.Names[0], destFileNtfsRef.IndexInMft);
// if not, track it
ntfsHardlinkTracker.Add(sourceFileNtfsRef.IndexInMft,
destFileNtfsRef.IndexInMft);
}
}
if (!skipCopying)
{
Logger.Info("[{0}/{1}] cp-file: {2}", i, taskId, destFile.FullName);
CloneStream(i, taskId, sourceFileStream, destFileStream);
}
// clone basic file informationsdestFile
Logger.Verbose("[{0}/{1}] cp-meta: {2}", i, taskId, destFile.FullName);
destFile.Attributes = sourceFile.Attributes;
destFile.CreationTime = sourceFile.CreationTime;
destFile.CreationTimeUtc = sourceFile.CreationTimeUtc;
destFile.IsReadOnly = sourceFile.IsReadOnly;
destFile.LastAccessTime = sourceFile.LastAccessTime;
destFile.LastAccessTimeUtc = sourceFile.LastAccessTimeUtc;
destFile.LastWriteTime = sourceFile.LastWriteTime;
destFile.LastWriteTimeUtc = sourceFile.LastWriteTimeUtc;
// file-system based cloning
if (destPartitionFS is NtfsFileSystem)
{
Logger.Verbose("[{0}/{1}] cp-fsec: {2}", i, taskId, destFile.FullName);
var ntfsSourceFS = (NtfsFileSystem)srcPartitionFS;
var ntfsDestFS = (NtfsFileSystem)destPartitionFS;
var destFileNtfsEntry = ntfsDestFS.GetDirectoryEntry(
destFile.FullName);
var destFileNtfsRef = ntfsDestFS.GetFile(
destFileNtfsEntry.Reference);
// clone security descriptors
var sourceNtfsSecurity = ntfsSourceFS.GetSecurity(
sourceFile.FullName);
ntfsDestFS.SetSecurity(destFile.FullName, sourceNtfsSecurity);
// clone short names if destination file is not a hard link
if (destFileNtfsEntry.Details.FileNameNamespace !=
FileNameNamespace.Posix || !destFileNtfsRef.HasWin32OrDosName)
{
Logger.Verbose("[{0}/{1}] cp-shrt: {2}", i, taskId, destFile.FullName);
var sourceNtfsShortName = ntfsSourceFS.GetShortName(
sourceFile.FullName);
if (sourceNtfsShortName != null)
{
ntfsSourceFS.SetShortName(destFile.FullName,
sourceNtfsShortName);
}
}
}
}
// advance recursion into directories
foreach (var sourceDirectory in sourceDir.GetDirectories())
{
if (srcPartitionFS is FatFileSystem)
{
// Don't copy SYSTEM~1 on FAT. Just don't do it...
if (sourceDirectory.Name.Equals("SYSTEM~1"))
{
continue;
}
}
cloneSrcFsToDestFs(sourceDirectory,
new DiscDirectoryInfo(
destPartitionFS,
Path.Combine(destDir.FullName, sourceDirectory.Name)
)
);
}
});
cloneSrcFsToDestFs(srcPartitionFS.Root, destPartitionFS.Root);
}
}
return(SUCCESS);
}
public static int Run(Options opts)
{
RunHelpers(opts);
Logger.Info("Opening image \"{0}\"", opts.Path);
var imageStream = OpenPath(opts.Path, FileMode.Open, FileAccess.Read, FileShare.None);
if (imageStream == null)
{
Logger.Error("Failed to open image!");
WaitForUserExit();
return(INVALID_ARGUMENT);
}
var bufferSize = opts.BlockCount * BLOCK_SIZE;
var buffer = new byte[bufferSize];
var writeLock = new object();
var lastPositionReportLock = new object();
var lastPositionReport = DateTime.Now;
Logger.Info("Starting deep scan of {0} (0x{1:X})", FormatBytes(imageStream.Length, 3), imageStream.Length);
while (imageStream.Position < imageStream.Length)
{
var read = imageStream.Read(buffer, 0, buffer.Length);
if (read != buffer.Length)
{
Logger.Error("Failed to read {0} of data, got {1} at position 0x{2:X}",
FormatBytes(buffer.Length, 3), FormatBytes(read, 3), imageStream.Position);
break;
}
Parallel.For(0, bufferSize - BLOCK_SIZE + 1, new ParallelOptions()
{
MaxDegreeOfParallelism = opts.Threads
},
(i, l) =>
{
var actualPosition = imageStream.Position - buffer.Length + i;
if (i % opts.ScanGranularity != 0)
{
return;
}
Stream stream = new MemoryStream(BLOCK_SIZE);
stream.Write(buffer, i, (stream as MemoryStream).Capacity);
// assume file system is on the rest of the stream to fix file table position validating
stream = new FixedLengthStream(stream, imageStream.Length - imageStream.Position, false);
if (opts.ScanForNtfs && NtfsFileSystem.Detect(stream))
{
lock (writeLock)
Logger.Info("Found NTFS file system at offset 0x{0:X} ({1})", actualPosition, FormatBytes(actualPosition, 3));
}
else if (opts.ScanForFat && FatFileSystem.Detect(stream))
{
lock (writeLock)
Logger.Info("Found FAT file system at offset 0x{0:X} ({1})", actualPosition, FormatBytes(actualPosition, 3));
}
else if (opts.ScanForExt && ExtFileSystem.Detect(stream))
{
lock (writeLock)
Logger.Info("Found EXT file system at offset 0x{0:X} ({1})", actualPosition, FormatBytes(actualPosition, 3));
}
lock (lastPositionReportLock)
{
var now = DateTime.Now;
if (now.Subtract(lastPositionReport).TotalSeconds >= 10.0)
{
lock (writeLock)
Logger.Info("Currently scanning at offset 0x{0:X} ({1})", actualPosition, FormatBytes(actualPosition, 3));
lastPositionReport = now;
}
}
});
}
Cleanup(imageStream);
WaitForUserExit();
return(SUCCESS);
}