/// <summary>A compare function that lists the differences between two TOC files</summary>
/// <param name="Other">The TOC class to compare against</param>
/// <remarks>The files referenced in the newly created TOC are compared against the files referenced in the old TOC. All files in both TOCs have their SHA256 checksums compared,
/// then files that no longer exist are listed, then new files that did not originally exist are listed.</remarks>
public void Compare( TOC Other )
{
// Work out which files have different SHA256 hash values
List<string> HashDifferences =
(
from Entry in Entries
from OtherEntry in Other.Entries
where Entry.Name == OtherEntry.Name
where Entry.Hash != OtherEntry.Hash
select Entry.Name
).ToList();
if( HashDifferences.Count == 0 )
{
Distill.Log( "No files with mismatched hash values", ConsoleColor.Green );
}
else
{
Distill.Log( HashDifferences.Count + " file(s) with mismatched hash values", ConsoleColor.Red );
HashDifferences.ForEach( x => Distill.Log( " ... mismatched hash: " + x.ToString(), ConsoleColor.Red ) );
}
// Work out which files do not currently exist
List<string> MissingFiles =
(
from Entry in Entries
where !Other.Entries.Contains( Entry )
select Entry.Name
).ToList();
if( MissingFiles.Count == 0 )
{
Distill.Log( "No missing files", ConsoleColor.Green );
}
else
{
Distill.Log( MissingFiles.Count + " missing file(s)", ConsoleColor.Red );
MissingFiles.ForEach( x => Distill.Log( " ... missing: " + x.ToString(), ConsoleColor.Red ) );
}
// Work out which files have been created
List<string> NewFiles =
(
from Entry in Other.Entries
where !Entries.Contains( Entry )
select Entry.Name
).ToList();
if( NewFiles.Count == 0 )
{
Distill.Log( "No new files", ConsoleColor.Green );
}
else
{
Distill.Log( NewFiles.Count + " new file(s)", ConsoleColor.Red );
NewFiles.ForEach( x => Distill.Log( " ... new: " + x.ToString(), ConsoleColor.Red ) );
}
}
/// <summary>A compare function that lists the differences between two TOC files</summary>
/// <param name="Other">The TOC class to compare against</param>
/// <remarks>The files referenced in the newly created TOC are compared against the files referenced in the old TOC. All files in both TOCs have their SHA256 checksums compared,
/// then files that no longer exist are listed, then new files that did not originally exist are listed.</remarks>
public void Compare(TOC Other)
{
// Work out which files have different SHA256 hash values
List <string> HashDifferences =
(
from Entry in Entries
from OtherEntry in Other.Entries
where Entry.Name == OtherEntry.Name
where Entry.Hash != OtherEntry.Hash
select Entry.Name
).ToList();
if (HashDifferences.Count == 0)
{
Distill.Log("No files with mismatched hash values", ConsoleColor.Green);
}
else
{
Distill.Log(HashDifferences.Count + " file(s) with mismatched hash values", ConsoleColor.Red);
HashDifferences.ForEach(x => Distill.Log(" ... mismatched hash: " + x.ToString(), ConsoleColor.Red));
}
// Work out which files do not currently exist
List <string> MissingFiles =
(
from Entry in Entries
where !Other.Entries.Contains(Entry)
select Entry.Name
).ToList();
if (MissingFiles.Count == 0)
{
Distill.Log("No missing files", ConsoleColor.Green);
}
else
{
Distill.Log(MissingFiles.Count + " missing file(s)", ConsoleColor.Red);
MissingFiles.ForEach(x => Distill.Log(" ... missing: " + x.ToString(), ConsoleColor.Red));
}
// Work out which files have been created
List <string> NewFiles =
(
from Entry in Other.Entries
where !Entries.Contains(Entry)
select Entry.Name
).ToList();
if (NewFiles.Count == 0)
{
Distill.Log("No new files", ConsoleColor.Green);
}
else
{
Distill.Log(NewFiles.Count + " new file(s)", ConsoleColor.Red);
NewFiles.ForEach(x => Distill.Log(" ... new: " + x.ToString(), ConsoleColor.Red));
}
}
/// <summary>Copy all the files in the TOC to the destination folder(s)</summary>
/// <param name="TableOfContents">Container for all the files to copy</param>
public static void CopyFiles(TOC TableOfContents)
{
if (Options.Destination.Length > 0)
{
long TotalFileCount = 0;
long TotalBytesCopied = 0;
bCopyingFiles = true;
Thread CopyFilesThread = new Thread(CopyFilesThreadProc);
CopyFilesThread.Start();
bVerifyingFiles = true;
Thread VerifyFilesThread = new Thread(VerifyFilesThreadProc);
VerifyFilesThread.Start();
foreach (TOCEntry Entry in TableOfContents.Entries)
{
// Handle file copies
FileInfo SourceFile = Entry.Info;
FileInfo DestFile = new FileInfo(Path.Combine(Options.Destination, Entry.Name));
if (CopyRequired(SourceFile, DestFile))
{
// Create copy packet and add to queue
CopyFilePacket Packet = new CopyFilePacket(SourceFile, DestFile, Entry.Hash);
CopyFilePackets.Enqueue(Packet);
TotalFileCount++;
TotalBytesCopied += SourceFile.Length;
}
else
{
DetailLog(" ... not copying: " + DestFile.FullName, ConsoleColor.DarkYellow);
}
}
while (CopyFilePackets.Count > 0 || VerifyFilePackets.Count > 0)
{
if (!Options.bLog)
{
ProgressLog(" ... waiting for " + CopyFilePackets.Count + " copies, and " + VerifyFilePackets.Count + " verifies ", ConsoleColor.Cyan);
}
Thread.Sleep(100);
}
if (!Options.bLog)
{
ProgressLog(" ... waiting for " + CopyFilePackets.Count + " copies, and " + VerifyFilePackets.Count + " verifies ", ConsoleColor.Cyan);
}
bCopyingFiles = false;
bVerifyingFiles = false;
Log("", ConsoleColor.Green);
Log("Completed copying " + TotalFileCount + " files totaling " + TotalBytesCopied + " bytes", ConsoleColor.Green);
}
}
/// <summary>Copy all the files in the TOC to the destination folder(s)</summary>
/// <param name="TableOfContents">Container for all the files to copy</param>
public static void CopyFiles( TOC TableOfContents )
{
if( Options.Destination.Length > 0 )
{
long TotalFileCount = 0;
long TotalBytesCopied = 0;
bCopyingFiles = true;
Thread CopyFilesThread = new Thread( CopyFilesThreadProc );
CopyFilesThread.Start();
bVerifyingFiles = true;
Thread VerifyFilesThread = new Thread( VerifyFilesThreadProc );
VerifyFilesThread.Start();
foreach( TOCEntry Entry in TableOfContents.Entries )
{
// Handle file copies
FileInfo SourceFile = Entry.Info;
FileInfo DestFile = new FileInfo( Path.Combine( Options.Destination, Entry.Name ) );
if( CopyRequired( SourceFile, DestFile ) )
{
// Create copy packet and add to queue
CopyFilePacket Packet = new CopyFilePacket( SourceFile, DestFile, Entry.Hash );
CopyFilePackets.Enqueue( Packet );
TotalFileCount++;
TotalBytesCopied += SourceFile.Length;
}
else
{
DetailLog( " ... not copying: " + DestFile.FullName, ConsoleColor.DarkYellow );
}
}
while( CopyFilePackets.Count > 0 || VerifyFilePackets.Count > 0 )
{
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + CopyFilePackets.Count + " copies, and " + VerifyFilePackets.Count + " verifies ", ConsoleColor.Cyan );
}
Thread.Sleep( 100 );
}
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + CopyFilePackets.Count + " copies, and " + VerifyFilePackets.Count + " verifies ", ConsoleColor.Cyan );
}
bCopyingFiles = false;
bVerifyingFiles = false;
Log( "", ConsoleColor.Green );
Log( "Completed copying " + TotalFileCount + " files totaling " + TotalBytesCopied + " bytes", ConsoleColor.Green );
}
}
/// <summary>Authenticate the local files against the checksums in the TOC</summary>
/// <param name="TableOfContents">Freshly created TOC from the local file system</param>
public static void Authenticate( TOC TableOfContents )
{
string TOCFileName = Path.Combine( Options.Source, Options.Game, "TOC.xml" );
TOC OriginalTableOfContents = XmlHandler.ReadXml<TOC>( TOCFileName );
if( OriginalTableOfContents.Entries != null )
{
OriginalTableOfContents.Compare( TableOfContents );
}
else
{
Error( "Failed to load TOC: " + TOCFileName );
}
}
/// <summary>Authenticate the local files against the checksums in the TOC</summary>
/// <param name="TableOfContents">Freshly created TOC from the local file system</param>
public static void Authenticate(TOC TableOfContents)
{
string TOCFileName = Path.Combine(Options.Source, Options.Game, "TOC.xml");
TOC OriginalTableOfContents = XmlHandler.ReadXml <TOC>(TOCFileName);
if (OriginalTableOfContents.Entries != null)
{
OriginalTableOfContents.Compare(TableOfContents);
}
else
{
Error("Failed to load TOC: " + TOCFileName);
}
}
/// <summary>
///
/// </summary>
/// <param name="TableOfContents"></param>
public static void FTPFiles( TOC TableOfContents )
{
if( Options.FTPSite.Length > 0 && Options.FTPUser.Length > 0 )
{
int TotalFilesUploaded = TableOfContents.Entries.Count;
TotalBytesUploaded = 0;
bCopyingFiles = true;
Thread FTPFilesThread = new Thread( FTPFilesThreadProc );
FTPFilesThread.Start();
foreach( TOCEntry Entry in TableOfContents.Entries )
{
// Handle file copies
FileInfo SourceFile = Entry.Info;
string DestFile = Path.Combine( Options.FTPFolder, Entry.Name ).Replace( "\\", "/" );
// Create copy packet and add to queue
FTPFilePacket Packet = new FTPFilePacket( SourceFile, DestFile );
FTPFilePackets.Enqueue( Packet );
}
while( FTPFilePackets.Count > 0 )
{
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + FTPFilePackets.Count + " FTPs (" + TotalBytesUploaded + " uploaded) ", ConsoleColor.Cyan );
}
Thread.Sleep( 100 );
}
if( !Options.bLog )
{
ProgressLog( " ... waiting for " + FTPFilePackets.Count + " FTPs (" + TotalBytesUploaded + " bytes uploaded) ", ConsoleColor.Cyan );
}
bFTPingFiles = false;
Log( "", ConsoleColor.Green );
Log( "Completed FTPing " + TotalFilesUploaded + " files, totaling " + TotalBytesUploaded + " bytes", ConsoleColor.Green );
}
}
/// <summary>
///
/// </summary>
/// <param name="TableOfContents"></param>
public static void FTPFiles(TOC TableOfContents)
{
if (Options.FTPSite.Length > 0 && Options.FTPUser.Length > 0)
{
int TotalFilesUploaded = TableOfContents.Entries.Count;
TotalBytesUploaded = 0;
bCopyingFiles = true;
Thread FTPFilesThread = new Thread(FTPFilesThreadProc);
FTPFilesThread.Start();
foreach (TOCEntry Entry in TableOfContents.Entries)
{
// Handle file copies
FileInfo SourceFile = Entry.Info;
string DestFile = Path.Combine(Options.FTPFolder, Entry.Name).Replace("\\", "/");
// Create copy packet and add to queue
FTPFilePacket Packet = new FTPFilePacket(SourceFile, DestFile);
FTPFilePackets.Enqueue(Packet);
}
while (FTPFilePackets.Count > 0)
{
if (!Options.bLog)
{
ProgressLog(" ... waiting for " + FTPFilePackets.Count + " FTPs (" + TotalBytesUploaded + " uploaded) ", ConsoleColor.Cyan);
}
Thread.Sleep(100);
}
if (!Options.bLog)
{
ProgressLog(" ... waiting for " + FTPFilePackets.Count + " FTPs (" + TotalBytesUploaded + " bytes uploaded) ", ConsoleColor.Cyan);
}
bFTPingFiles = false;
Log("", ConsoleColor.Green);
Log("Completed FTPing " + TotalFilesUploaded + " files, totaling " + TotalBytesUploaded + " bytes", ConsoleColor.Green);
}
}
/// <summary>Copy all the files in the TOC to the all the destination zips</summary>
/// <param name="TableOfContents">Container for all the files to copy</param>
public static void ZipFiles( TOC TableOfContents )
{
// Handle zips
if( Options.ZipName.Length > 0 )
{
long TotalFilesZipped = TableOfContents.Entries.Count;
long TotalBytesZipped = TableOfContents.Entries.Sum( x => x.Info.Length );
ZipFile Zip = new ZipFile( Options.ZipName );
Zip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level9;
Zip.UseZip64WhenSaving = Zip64Option.Always;
Zip.BufferSize = 0x10000;
TableOfContents.Entries.ForEach( x => Zip.UpdateFile( x.Name ) );
Log( " ... saving zip: " + Zip.Name, ConsoleColor.Green );
Zip.Save();
FileInfo ZipInfo = new FileInfo( Zip.Name );
Log( "Completed saving zip with " + TotalFilesZipped + " files to " + ZipInfo.Length + " bytes (from " + TotalBytesZipped + ")", ConsoleColor.Green );
}
}
/// <summary>Copy all the files in the TOC to the all the destination zips</summary>
/// <param name="TableOfContents">Container for all the files to copy</param>
public static void ZipFiles(TOC TableOfContents)
{
// Handle zips
if (Options.ZipName.Length > 0)
{
long TotalFilesZipped = TableOfContents.Entries.Count;
long TotalBytesZipped = TableOfContents.Entries.Sum(x => x.Info.Length);
ZipFile Zip = new ZipFile(Options.ZipName);
Zip.CompressionLevel = Ionic.Zlib.CompressionLevel.Level9;
Zip.UseZip64WhenSaving = Zip64Option.Always;
Zip.BufferSize = 0x10000;
TableOfContents.Entries.ForEach(x => Zip.UpdateFile(x.Name));
Log(" ... saving zip: " + Zip.Name, ConsoleColor.Green);
Zip.Save();
FileInfo ZipInfo = new FileInfo(Zip.Name);
Log("Completed saving zip with " + TotalFilesZipped + " files to " + ZipInfo.Length + " bytes (from " + TotalBytesZipped + ")", ConsoleColor.Green);
}
}
/// <summary>The main entry point of the program</summary>
/// <param name="Arguments">The arguments passed on on the commandline</param>
/// <returns>Zero on success, non zero for failure</returns>
private static int Main(string[] Arguments)
{
if (Arguments.Length == 0)
{
ShowUsage();
return(1);
}
// Set the current directory to the root of the branch
Environment.CurrentDirectory = Path.Combine(Environment.CurrentDirectory, "..", "..", "..");
Options.Source = Environment.CurrentDirectory;
Options.Destination = Environment.CurrentDirectory;
// Remember console color for restoring on exit
ConsoleColor OriginalConsoleColor = Console.ForegroundColor;
// Parse the command line for settings
ParseArguments(Arguments);
// Load the Distill.xml file and the game specific version
string SettingsLocation = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), Options.DistillSettings);
if (File.Exists(SettingsLocation) == false)
{
Error("Unable to find distill file " + Options.DistillSettings);
Console.ForegroundColor = OriginalConsoleColor;
return(-1);
}
DistillSettings Settings = XmlHandler.ReadXml <DistillSettings>(SettingsLocation);
// Find the known languages based on the folders in Engine Localization
Settings.KnownLanguages = FindKnownLanguages();
// Ensure the settings are valid
if (!ValidateOptions(Settings))
{
Console.ForegroundColor = OriginalConsoleColor;
return(-1);
}
// Get the set of tags we wish to copy
TagSet Set = GetTagSet(Settings);
if (Set == null)
{
Console.ForegroundColor = OriginalConsoleColor;
return(-1);
}
// Get a list of filesets that includes all the unexpanded macros
List <FileSet> DecoratedFileSets = GetFileSets(Settings, Set);
if (DecoratedFileSets.Count == 0)
{
Error("No file sets for game, platform, tagset combination!");
Console.ForegroundColor = OriginalConsoleColor;
return(-1);
}
// Expand out all the macros
List <FileSet> FileSets = ExpandMacros(DecoratedFileSets, Settings);
// Get the files referenced by the filesets
List <FileInfo> Files = GetFiles(FileSets);
// Create a TOC
TOC TableOfContents = new TOC(Files);
Files = null;
if (Options.bAuthenticate)
{
// If we're authenticating, compare the newly created TOC from files with the existing one on disk
Authenticate(TableOfContents);
}
else
{
// Only write a TOC if we're copying locally to somewhere else (the most common case)
string TOCFileName = Path.Combine(Options.Source, Options.Game, "TOC.xml");
if (Options.bTOC && (Options.Source == Environment.CurrentDirectory))
{
Log(" ... writing TOC: " + TOCFileName, ConsoleColor.Cyan);
XmlHandler.WriteXml <TOC>(TableOfContents, TOCFileName, "");
}
// Copy the TOC if it exists
if (Options.bTOC)
{
FileInfo TOCFileInfo = new FileInfo(TOCFileName);
if (TOCFileInfo.Exists)
{
TableOfContents.Entries.Add(new TOCEntry(TOCFileInfo));
}
else
{
Error(" ... Expected a TOC but there isn't one to copy: " + TOCFileName);
}
}
// Copy files
ZipFiles(TableOfContents);
CopyFiles(TableOfContents);
FTPFiles(TableOfContents);
Log("Distill process successful!", ConsoleColor.Green);
}
// Restore console color
Console.ForegroundColor = OriginalConsoleColor;
return(ErrorCode);
}
/// <summary>The main entry point of the program</summary>
/// <param name="Arguments">The arguments passed on on the commandline</param>
/// <returns>Zero on success, non zero for failure</returns>
private static int Main( string[] Arguments )
{
if( Arguments.Length == 0 )
{
ShowUsage();
return 1;
}
// Set the current directory to the root of the branch
Environment.CurrentDirectory = Path.Combine( Environment.CurrentDirectory, "..", "..", ".." );
Options.Source = Environment.CurrentDirectory;
Options.Destination = Environment.CurrentDirectory;
// Remember console color for restoring on exit
ConsoleColor OriginalConsoleColor = Console.ForegroundColor;
// Parse the command line for settings
ParseArguments( Arguments );
// Load the Distill.xml file and the game specific version
string SettingsLocation = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), Options.DistillSettings);
if (File.Exists(SettingsLocation) == false)
{
Error("Unable to find distill file " + Options.DistillSettings);
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
DistillSettings Settings = XmlHandler.ReadXml<DistillSettings>( SettingsLocation );
// Find the known languages based on the folders in Engine Localization
Settings.KnownLanguages = FindKnownLanguages();
// Ensure the settings are valid
if( !ValidateOptions( Settings ) )
{
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
// Get the set of tags we wish to copy
TagSet Set = GetTagSet( Settings );
if( Set == null )
{
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
// Get a list of filesets that includes all the unexpanded macros
List<FileSet> DecoratedFileSets = GetFileSets( Settings, Set );
if( DecoratedFileSets.Count == 0 )
{
Error( "No file sets for game, platform, tagset combination!" );
Console.ForegroundColor = OriginalConsoleColor;
return -1;
}
// Expand out all the macros
List<FileSet> FileSets = ExpandMacros( DecoratedFileSets, Settings );
// Get the files referenced by the filesets
List<FileInfo> Files = GetFiles( FileSets );
// Create a TOC
TOC TableOfContents = new TOC( Files );
Files = null;
if( Options.bAuthenticate )
{
// If we're authenticating, compare the newly created TOC from files with the existing one on disk
Authenticate( TableOfContents );
}
else
{
// Only write a TOC if we're copying locally to somewhere else (the most common case)
string TOCFileName = Path.Combine( Options.Source, Options.Game, "TOC.xml" );
if( Options.bTOC && (Options.Source == Environment.CurrentDirectory) )
{
Log( " ... writing TOC: " + TOCFileName, ConsoleColor.Cyan );
XmlHandler.WriteXml<TOC>( TableOfContents, TOCFileName, "" );
}
// Copy the TOC if it exists
if (Options.bTOC)
{
FileInfo TOCFileInfo = new FileInfo(TOCFileName);
if (TOCFileInfo.Exists)
{
TableOfContents.Entries.Add(new TOCEntry(TOCFileInfo));
}
else
{
Error(" ... Expected a TOC but there isn't one to copy: " + TOCFileName);
}
}
// Copy files
ZipFiles( TableOfContents );
CopyFiles( TableOfContents );
FTPFiles( TableOfContents );
Log( "Distill process successful!", ConsoleColor.Green );
}
// Restore console color
Console.ForegroundColor = OriginalConsoleColor;
return ErrorCode;
}
