public static void Log(ELogType logType, string format, params object[] args)
{
if (_callback != null)
{
string message = string.Format(format, args);
_callback.Invoke(logType, message);
}
}
private void Recv(IAsyncResult res)
{
var bytesRecv = 0;
try
{
bytesRecv = _udpSocket.EndReceive(res);
}
catch (ObjectDisposedException)
{
return;
}
if (bytesRecv > _headerSize)
{
var logLine = Encoding.UTF8.GetString(_recvData, _headerSize, bytesRecv - _headerSize);
Callback?.Invoke(string.Copy(logLine));
foreach (var i in _eventsInstanceList)
{
i.ProcessLog(string.Copy(logLine));
}
}
_udpSocket.BeginReceive(_recvData, 0, _recvData.Length, SocketFlags.None, Recv, null);
}
/// <summary>
/// A function that can contain code for logging or debugging returned results.
/// Intended to be used when returning a non-zero Result:
/// <code>return result.Log();</code>
/// </summary>
/// <returns>The called <see cref="Result"/> value.</returns>
public Result Log()
{
#if DEBUG
LogCallback?.Invoke(this);
#endif
return(this);
}
static internal void SendMessage(string message, MessageType type)
{
if ((type & m_allowedTypes) != 0)
{
m_log?.Invoke(message, type);
}
}
internal static void ExecuteCallback(LogLevel level, string message, bool containsPii)
{
lock (LockObj)
{
_logCallback?.Invoke(level, message, containsPii);
}
}
public void ImportAllJsonToAsset(string inputAssetsPath, string jsonDirectory, string outputAssetsPath)
{
using (AssetsFile file = AssetsFile.Open(inputAssetsPath))
{
var assetSerializer = new UnitySerializer(file);
foreach (DataType dType in types)
{
AssetInfo info = file.GetAssetByName(dType.AssetName);
string jsonPath = Path.Combine(jsonDirectory, $"{dType.AssetName}.json");
LogCallback?.Invoke(dType.AssetName);
var obj = skylessSerializer.DeserializeJson(dType.TypeName, jsonPath);
var binaryData = skylessSerializer.SerializeBinary(dType.TypeName, obj);
var textAsset = new TextAsset();
textAsset.m_Name = dType.AssetName;
textAsset.m_Data = binaryData;
var assetData = assetSerializer.Serialize(textAsset);
file.ReplaceAsset(info.pathID, assetData);
}
file.Save(outputAssetsPath);
}
}
private static void CallLogCallback(string logString, string stackTrace, LogType type, bool invokedOnMainThread)
{
if (invokedOnMainThread)
{
s_LogCallbackHandler?.Invoke(logString, stackTrace, type);
}
s_LogCallbackHandlerThreaded?.Invoke(logString, stackTrace, type);
}
public void WriteLine(string line)
{
lock (s_lock)
{
_logFile?.WriteLine(line);
_logFile?.Flush();
LogCallback?.Invoke(line);
}
}
private void Log(LogLevel msalLogLevel, string messageWithPii, string messageScrubbed)
{
if (_loggingCallback == null || msalLogLevel > _logLevel)
{
return;
}
//format log message;
string correlationId = CorrelationId.Equals(Guid.Empty)
? string.Empty
: " - " + CorrelationId;
var msalIdParameters = MsalIdHelper.GetMsalIdParameters(this);
string os = "N/A";
if (msalIdParameters.TryGetValue(MsalIdParameter.OS, out string osValue))
{
os = osValue;
}
bool messageWithPiiExists = !string.IsNullOrWhiteSpace(messageWithPii);
// If we have a message with PII, and PII logging is enabled, use the PII message, else use the scrubbed message.
bool isLoggingPii = messageWithPiiExists && PiiLoggingEnabled;
string messageToLog = isLoggingPii ? messageWithPii : messageScrubbed;
string log = string.Format(CultureInfo.InvariantCulture, "{0} MSAL {1} {2} {3} [{4}{5}]{6} {7}",
isLoggingPii ? "(True)" : "(False)",
MsalIdHelper.GetMsalVersion(),
msalIdParameters[MsalIdParameter.Product],
os, DateTime.UtcNow, correlationId, ClientInformation, messageToLog);
if (_isDefaultPlatformLoggingEnabled)
{
switch (msalLogLevel)
{
case LogLevel.Error:
_platformLogger.Error(log);
break;
case LogLevel.Warning:
_platformLogger.Warning(log);
break;
case LogLevel.Info:
_platformLogger.Information(log);
break;
case LogLevel.Verbose:
_platformLogger.Verbose(log);
break;
}
}
_loggingCallback.Invoke(msalLogLevel, log, isLoggingPii);
}
public TrrntZipStatus Process(FileInfo fi)
{
if (Program.VerboseLogging)
{
StatusLogCallBack?.Invoke(ThreadID, "");
}
StatusLogCallBack?.Invoke(ThreadID, fi.Name + " - ");
// First open the zip (7z) file, and fail out if it is corrupt.
ICompress zipFile;
TrrntZipStatus tzs = OpenZip(fi, out zipFile);
// this will return ValidTrrntZip or CorruptZip.
if ((tzs & TrrntZipStatus.CorruptZip) == TrrntZipStatus.CorruptZip)
{
StatusLogCallBack?.Invoke(ThreadID, "Zip file is corrupt");
return(TrrntZipStatus.CorruptZip);
}
// the zip file may have found a valid trrntzip header, but we now check that all the file info
// is actually valid, and may invalidate it being a valid trrntzip if any problem is found.
List <ZippedFile> zippedFiles = ReadZipContent(zipFile);
tzs |= TorrentZipCheck.CheckZipFiles(ref zippedFiles);
// if tza is now just 'ValidTrrntzip' the it is fully valid, and nothing needs to be done to it.
if (tzs == TrrntZipStatus.ValidTrrntzip && !Program.ForceReZip || Program.CheckOnly)
{
StatusLogCallBack?.Invoke(ThreadID, "Skipping File");
return(TrrntZipStatus.ValidTrrntzip);
}
StatusLogCallBack?.Invoke(ThreadID, "TorrentZipping");
TrrntZipStatus fixedTzs = TorrentZipRebuild.ReZipFiles(zippedFiles, zipFile, _buffer, StatusCallBack, StatusLogCallBack, ThreadID);
return(fixedTzs);
}
/// <summary>
/// This method is invoked after each step/substep of the glTF import has
/// completed, in order to update the progress log accordingly.
/// </summary>
public void OnImportProgress(ImportStep importStep, int numCompleted, int total)
{
UpdateProgress(importStep, numCompleted, total);
string message = GetProgressMessage();
// Update existing tail log line if we are still importing
// the same type of glTF entity (e.g. textures), or
// add a new line if we have started to import
// a new type.
if (IsNewImportStep())
{
AddLineCallback?.Invoke(message);
}
else
{
UpdateLineCallback?.Invoke(message);
}
}
public void ExportAllToJson(string resourceAssetsPath, string outputDirectory)
{
using (AssetsFile file = AssetsFile.Open(resourceAssetsPath))
{
var assetSerializer = new UnitySerializer(file);
foreach (DataType dType in types)
{
LogCallback?.Invoke(dType.AssetName);
AssetInfo info = file.GetAssetByName(dType.AssetName);
var textAsset = assetSerializer.Deserialize <TextAsset>(info);
var obj = skylessSerializer.DeserializeBinary(dType.TypeName, textAsset.m_Data);
string jsonPath = Path.Combine(outputDirectory, $"{dType.AssetName}.json");
skylessSerializer.SerializeJson(jsonPath, obj);
}
}
}
private void Log(LogLevel authLogLevel, string messageWithPii, string messageScrubbed)
{
if (_loggingCallback == null || authLogLevel >= _logLevel)
{
return;
}
bool messageWithPiiExists = !string.IsNullOrWhiteSpace(messageWithPii);
// If we have a message with PII, and PII logging is enabled, use the PII message, else use the scrubbed message.
bool isLoggingPii = messageWithPiiExists && PiiLoggingEnabled;
string messageToLog = isLoggingPii ? messageWithPii : messageScrubbed;
string log = $"{isLoggingPii} AUTH {DateTime.UtcNow} {ClientInformation} {messageToLog}";
if (_isDefaultPlatformLoggingEnabled)
{
switch (authLogLevel)
{
case LogLevel.Error:
_platformLogger.Error(log);
break;
case LogLevel.Warning:
_platformLogger.Warning(log);
break;
case LogLevel.Info:
_platformLogger.Information(log);
break;
case LogLevel.Verbose:
_platformLogger.Verbose(log);
break;
}
}
_loggingCallback.Invoke(authLogLevel, log, isLoggingPii);
}
public static TrrntZipStatus ReZipFiles(List <ZippedFile> zippedFiles, ICompress originalZipFile, byte[] buffer, StatusCallback statusCallBack, LogCallback logCallback, int threadId)
{
zipType inputType;
switch (originalZipFile)
{
case Zip _:
inputType = zipType.zip;
break;
case SevenZ _:
inputType = zipType.sevenzip;
break;
case Compress.File.File _:
inputType = zipType.iso;
break;
default:
return(TrrntZipStatus.Unknown);
}
zipType outputType = Program.OutZip == zipType.both ? inputType : Program.OutZip;
if (outputType == zipType.iso)
{
outputType = zipType.zip;
}
int bufferSize = buffer.Length;
string filename = originalZipFile.ZipFilename;
string tmpFilename = Path.Combine(Path.GetDirectoryName(filename), Path.GetFileNameWithoutExtension(filename) + ".tmp");
string outExt = outputType == zipType.zip ? ".zip" : ".7z";
string outfilename = Path.Combine(Path.GetDirectoryName(filename), Path.GetFileNameWithoutExtension(filename) + outExt);
if (inputType != outputType)
{
if (File.Exists(outfilename))
{
logCallback?.Invoke(threadId, "Error output " + outExt + " file already exists");
return(TrrntZipStatus.RepeatFilesFound);
}
}
if (File.Exists(tmpFilename))
{
File.Delete(tmpFilename);
}
ICompress zipFileOut = outputType == zipType.zip ? new Zip() : (ICompress) new SevenZ();
try
{
zipFileOut.ZipFileCreate(tmpFilename);
ulong fileSizeTotal = 0;
ulong fileSizeProgress = 0;
int filePercentReported = 20;
foreach (ZippedFile f in zippedFiles)
{
fileSizeTotal += f.Size;
}
// by now the zippedFiles have been sorted so just loop over them
foreach (ZippedFile t in zippedFiles)
{
if (Program.VerboseLogging)
{
logCallback?.Invoke(threadId, $"{t.Size,15} {t.StringCRC} {t.Name}");
}
Stream readStream = null;
ulong streamSize = 0;
ZipReturn zrInput = ZipReturn.ZipUntested;
switch (originalZipFile)
{
case Zip z:
zrInput = z.ZipFileOpenReadStream(t.Index, false, out readStream, out streamSize, out ushort _);
break;
case SevenZ z7:
zrInput = z7.ZipFileOpenReadStream(t.Index, out readStream, out streamSize);
break;
case Compress.File.File zf:
zrInput = zf.ZipFileOpenReadStream(t.Index, out readStream, out streamSize);
break;
}
ZipReturn zrOutput = zipFileOut.ZipFileOpenWriteStream(false, true, t.Name, streamSize, 8, out Stream writeStream);
if ((zrInput != ZipReturn.ZipGood) || (zrOutput != ZipReturn.ZipGood))
{
//Error writing local File.
zipFileOut.ZipFileClose();
originalZipFile.ZipFileClose();
File.Delete(tmpFilename);
return(TrrntZipStatus.CorruptZip);
}
Stream crcCs = new CrcCalculatorStream(readStream, true);
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > (ulong)bufferSize ? bufferSize : (int)sizetogo;
fileSizeProgress += (ulong)sizenow;
int filePercent = (int)((double)fileSizeProgress / fileSizeTotal * 20);
if (filePercent != filePercentReported)
{
statusCallBack?.Invoke(threadId, filePercent * 5);
filePercentReported = filePercent;
}
crcCs.Read(buffer, 0, sizenow);
writeStream.Write(buffer, 0, sizenow);
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
crcCs.Close();
if (inputType != zipType.sevenzip)
{
originalZipFile.ZipFileCloseReadStream();
}
uint crc = (uint)((CrcCalculatorStream)crcCs).Crc;
if (t.CRC == null)
{
t.CRC = crc;
}
if (crc != t.CRC)
{
return(TrrntZipStatus.CorruptZip);
}
zipFileOut.ZipFileCloseWriteStream(t.ByteCRC);
}
statusCallBack?.Invoke(threadId, 100);
zipFileOut.ZipFileClose();
originalZipFile.ZipFileClose();
File.Delete(filename);
File.Move(tmpFilename, outfilename);
return(TrrntZipStatus.ValidTrrntzip);
}
catch (Exception)
{
zipFileOut?.ZipFileCloseFailed();
originalZipFile?.ZipFileClose();
return(TrrntZipStatus.CorruptZip);
}
}
static List <LogData> log_list = new List <LogData>(); // messages stored here
// these three are utility functions
// logs message of type INFO
static public void Info(LogSource s, string d)
{
log_list.Add(new LogData(DateTime.Now, LogOption.INFO, s, d));
OnLog?.Invoke(LogOption.INFO, s, d);
}
public static TrrntZipStatus CheckZipFiles(ref List <ZippedFile> zippedFiles, int ThreadID, LogCallback StatusLogCallBack)
{
TrrntZipStatus tzStatus = TrrntZipStatus.Unknown;
// ***************************** RULE 1 *************************************
// Directory separator should be a '/' a '\' is invalid and should be replaced with '/'
//
// check if any '\' = 92 need converted to '/' = 47
// this needs done before the sort, so that the sort is correct.
// return BadDirectorySeparator if errors found.
bool error1 = false;
foreach (ZippedFile t in zippedFiles)
{
char[] bytes = t.Name.ToCharArray();
bool fixDir = false;
for (int j = 0; j < bytes.Length; j++)
{
if (bytes[j] != 92)
{
continue;
}
fixDir = true;
bytes[j] = (char)47;
tzStatus |= TrrntZipStatus.BadDirectorySeparator;
if (!error1 && Program.VerboseLogging)
{
error1 = true;
StatusLogCallBack?.Invoke(ThreadID, "Incorrect directory separator found");
}
}
if (fixDir)
{
t.Name = new string(bytes);
}
}
// ***************************** RULE 2 *************************************
// All Files in a torrentzip should be sorted with a lower case file compare.
//
// if needed sort the files correctly, and return Unsorted if errors found.
bool error2 = false;
bool thisSortFound = true;
while (thisSortFound)
{
thisSortFound = false;
for (int i = 0; i < zippedFiles.Count - 1; i++)
{
int c = TrrntZipStringCompare(zippedFiles[i].Name, zippedFiles[i + 1].Name);
if (c > 0)
{
ZippedFile T = zippedFiles[i];
zippedFiles[i] = zippedFiles[i + 1];
zippedFiles[i + 1] = T;
tzStatus |= TrrntZipStatus.Unsorted;
thisSortFound = true;
if (!error2 && Program.VerboseLogging)
{
error2 = true;
StatusLogCallBack?.Invoke(ThreadID, "Incorrect file order found");
}
}
}
}
// ***************************** RULE 3 *************************************
// Directory marker files are only needed if they are empty directories.
//
// now that the files are sorted correctly, we can see if there are unneeded
// directory files, by first finding directory files (these end in a '\' character ascii 92)
// and then checking if the next file is a file in that found directory.
// If we find this 2 entry pattern (directory followed by file in that directory)
// then the directory entry should not be present and the torrentzip is incorrect.
// return ExtraDirectoryEnteries if error is found.
bool error3 = false;
for (int i = 0; i < zippedFiles.Count - 1; i++)
{
// check if this is a directory entry
if (zippedFiles[i].Name[zippedFiles[i].Name.Length - 1] != 47)
{
continue;
}
// check if the next filename is shorter or equal to this filename.
// if it is shorter or equal it cannot be a file in the directory.
if (zippedFiles[i + 1].Name.Length <= zippedFiles[i].Name.Length)
{
continue;
}
// check if the directory part of the two file enteries match
// if they do we found an incorrect directory entry.
bool delete = true;
for (int j = 0; j < zippedFiles[i].Name.Length; j++)
{
if (zippedFiles[i].Name[j] != zippedFiles[i + 1].Name[j])
{
delete = false;
break;
}
}
// we found an incorrect directory so remove it.
if (delete)
{
zippedFiles.RemoveAt(i);
tzStatus |= TrrntZipStatus.ExtraDirectoryEnteries;
if (!error3 && Program.VerboseLogging)
{
error3 = true;
StatusLogCallBack?.Invoke(ThreadID, "Un-needed directory records found");
}
i--;
}
}
// check for repeat files
bool error4 = false;
for (int i = 0; i < zippedFiles.Count - 1; i++)
{
if (zippedFiles[i].Name == zippedFiles[i + 1].Name)
{
tzStatus |= TrrntZipStatus.RepeatFilesFound;
if (!error4 && Program.VerboseLogging)
{
error4 = true;
StatusLogCallBack?.Invoke(ThreadID, "Duplcate file enteries found");
}
}
}
return(tzStatus);
}
public TrrntZipStatus Process(FileInfo fi)
{
if (Program.VerboseLogging)
{
StatusLogCallBack?.Invoke(ThreadId, "");
}
StatusLogCallBack?.Invoke(ThreadId, fi.Name + " - ");
// First open the zip (7z) file, and fail out if it is corrupt.
TrrntZipStatus tzs = OpenZip(fi, out ICompress zipFile);
// this will return ValidTrrntZip or CorruptZip.
for (int i = 0; i < zipFile.LocalFilesCount(); i++)
{
Debug.WriteLine("Name = " + zipFile.Filename(i) + " , " + zipFile.UncompressedSize(i));
}
if ((tzs & TrrntZipStatus.CorruptZip) == TrrntZipStatus.CorruptZip)
{
StatusLogCallBack?.Invoke(ThreadId, "Zip file is corrupt");
return(TrrntZipStatus.CorruptZip);
}
// the zip file may have found a valid trrntzip header, but we now check that all the file info
// is actually valid, and may invalidate it being a valid trrntzip if any problem is found.
List <ZippedFile> zippedFiles = ReadZipContent(zipFile);
// check if the compression type has changed
zipType inputType;
switch (zipFile)
{
case Zip _:
tzs |= TorrentZipCheck.CheckZipFiles(ref zippedFiles, ThreadId, StatusLogCallBack);
inputType = zipType.zip;
break;
case SevenZ _:
tzs |= TorrentZipCheck.CheckSevenZipFiles(ref zippedFiles, ThreadId, StatusLogCallBack);
inputType = zipType.sevenzip;
break;
case File _:
inputType = zipType.iso;
break;
default:
return(TrrntZipStatus.Unknown);
}
zipType outputType = Program.OutZip == zipType.both ? inputType : Program.OutZip;
if (outputType == zipType.iso)
{
outputType = zipType.zip;
}
bool compressionChanged = inputType != outputType;
// if tza is now just 'ValidTrrntzip' the it is fully valid, and nothing needs to be done to it.
if (((tzs == TrrntZipStatus.ValidTrrntzip) && !compressionChanged && !Program.ForceReZip) || Program.CheckOnly)
{
StatusLogCallBack?.Invoke(ThreadId, "Skipping File");
return(tzs);
}
// if compressionChanged then the required file order will also have changed to need to re-sort the files.
if (compressionChanged)
{
switch (outputType)
{
case zipType.zip:
tzs |= TorrentZipCheck.CheckZipFiles(ref zippedFiles, ThreadId, StatusLogCallBack);
break;
case zipType.sevenzip:
tzs |= TorrentZipCheck.CheckSevenZipFiles(ref zippedFiles, ThreadId, StatusLogCallBack);
break;
}
}
StatusLogCallBack?.Invoke(ThreadId, "TorrentZipping");
TrrntZipStatus fixedTzs = TorrentZipRebuild.ReZipFiles(zippedFiles, zipFile, _buffer, StatusCallBack, StatusLogCallBack, ThreadId);
return(fixedTzs);
}
public static TrrntZipStatus ReZipFiles(List <ZippedFile> zippedFiles, ICompress originalZipFile, byte[] buffer, StatusCallback StatusCallBack, LogCallback LogCallback, int ThreadID)
{
int bufferSize = buffer.Length;
string filename = originalZipFile.ZipFilename;
string tmpFilename = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(filename) + ".tmp";
string outfilename = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(filename) + ".zip";
if (Path.GetExtension(filename) == ".7z")
{
if (File.Exists(outfilename))
{
LogCallback?.Invoke(ThreadID, "Error output .zip file already exists");
return(TrrntZipStatus.RepeatFilesFound);
}
}
if (IO.File.Exists(tmpFilename))
{
IO.File.Delete(tmpFilename);
}
ICompress zipFileOut = new ZipFile();
try
{
zipFileOut.ZipFileCreate(tmpFilename);
// by now the zippedFiles have been sorted so just loop over them
for (int i = 0; i < zippedFiles.Count; i++)
{
StatusCallBack?.Invoke(ThreadID, (int)((double)(i + 1) / (zippedFiles.Count) * 100));
ZippedFile t = zippedFiles[i];
if (Program.VerboseLogging)
{
LogCallback?.Invoke(ThreadID, $"{t.Size,15} {t.StringCRC} {t.Name}");
}
Stream readStream = null;
ulong streamSize = 0;
ushort compMethod;
ZipFile z = originalZipFile as ZipFile;
ZipReturn zrInput = ZipReturn.ZipUntested;
if (z != null)
{
zrInput = z.ZipFileOpenReadStream(t.Index, false, out readStream, out streamSize, out compMethod);
}
SevenZ z7 = originalZipFile as SevenZ;
if (z7 != null)
{
zrInput = z7.ZipFileOpenReadStream(t.Index, out readStream, out streamSize);
}
Stream writeStream;
ZipReturn zrOutput = zipFileOut.ZipFileOpenWriteStream(false, true, t.Name, streamSize, 8, out writeStream);
if (zrInput != ZipReturn.ZipGood || zrOutput != ZipReturn.ZipGood)
{
//Error writing local File.
zipFileOut.ZipFileClose();
originalZipFile.ZipFileClose();
IO.File.Delete(tmpFilename);
return(TrrntZipStatus.CorruptZip);
}
Stream crcCs = new CrcCalculatorStream(readStream, true);
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > (ulong)bufferSize ? bufferSize : (int)sizetogo;
crcCs.Read(buffer, 0, sizenow);
writeStream.Write(buffer, 0, sizenow);
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
crcCs.Close();
if (z != null)
{
originalZipFile.ZipFileCloseReadStream();
}
uint crc = (uint)((CrcCalculatorStream)crcCs).Crc;
if (crc != t.CRC)
{
return(TrrntZipStatus.CorruptZip);
}
zipFileOut.ZipFileCloseWriteStream(t.ByteCRC);
}
zipFileOut.ZipFileClose();
originalZipFile.ZipFileClose();
IO.File.Delete(filename);
IO.File.Move(tmpFilename, outfilename);
return(TrrntZipStatus.ValidTrrntzip);
}
catch (Exception)
{
zipFileOut?.ZipFileCloseFailed();
originalZipFile?.ZipFileClose();
return(TrrntZipStatus.CorruptZip);
}
}
public virtual void Log(string text, LogLevels level, LogIndents indentBefore, LogIndents indentAfter)
{
LogCallback?.Invoke(text, level, indentBefore, indentAfter);
}
