public BLTEStream(Stream src, MD5Hash md5)
{
_stream = src;
_reader = new BinaryReader(src);
Parse(md5);
}
public BLTEStream(Stream source, MD5Hash hash)
{
stream = source;
reader = new BinaryReader(source);
Parse(hash);
}
public unsafe IndexEntry GetIndexInfo(MD5Hash key)
{
ulong* ptr = (ulong*)&key;
ptr[1] &= 0xFf;
IndexEntry result;
if (!LocalIndexData.TryGetValue(key, out result))
Log.Write("CASC: Missing index {0}", key.ToHexString());
return result;
}
public unsafe IndexEntry GetIndexInfo(MD5Hash key)
{
ulong* ptr = (ulong*)&key;
ptr[1] &= 0xFF;
IndexEntry result;
if (!LocalIndexData.TryGetValue(key, out result))
Logger.WriteLine("LocalIndexHandler: missing index: {0}", key.ToHexString());
return result;
}
protected Stream OpenFileOnlineInternal(IndexEntry idxInfo, MD5Hash key)
{
if (idxInfo != null)
{
Stream s = CDNIndex.OpenDataFile(idxInfo);
return new BLTEStream(s, key);
}
else
{
Stream s = CDNIndex.OpenDataFileDirect(key);
return new BLTEStream(s, key);
}
}
public Stream OpenFile(MD5Hash key)
{
try
{
if (Config.OnlineMode)
return OpenFileOnline(key);
else
return OpenFileLocal(key);
}
catch (Exception exc) when (!(exc is BLTEDecoderException))
{
return OpenFileOnline(key);
}
}
public EncodingHandler(BLTEStream blte)
{
if (blte.Length != long.Parse(CASContainer.BuildConfig["encoding-size"][0]))
{
CASContainer.Settings?.Logger.LogAndThrow(Logging.LogType.Critical, "Encoding File is corrupt.");
}
BinaryReader stream = new BinaryReader(blte);
Header = new EncodingHeader()
{
Magic = stream.ReadBytes(2),
Version = stream.ReadByte(),
ChecksumSizeA = stream.ReadByte(),
ChecksumSizeB = stream.ReadByte(),
FlagsA = stream.ReadUInt16(),
FlagsB = stream.ReadUInt16(),
NumEntriesA = stream.ReadUInt32BE(),
NumEntriesB = stream.ReadUInt32BE(),
StringBlockSize = stream.ReadUInt40BE()
};
// stringTableA
LayoutStringTable.AddRange(Encoding.ASCII.GetString(stream.ReadBytes((int)Header.StringBlockSize)).Split('\0'));
// skip header block A
stream.ReadBytes((int)Header.NumEntriesA * 32);
// encoding table entry block
for (int i = 0; i < Header.NumEntriesA; i++)
{
long start = stream.BaseStream.Position;
ushort keysCount;
while ((keysCount = stream.ReadUInt16()) != 0)
{
EncodingEntry entry = new EncodingEntry()
{
DecompressedSize = stream.ReadUInt32BE(),
Hash = new MD5Hash(stream)
};
for (int ki = 0; ki < keysCount; ki++)
{
entry.Keys.Add(new MD5Hash(stream));
}
Data.Add(entry.Hash, entry);
}
if (stream.BaseStream.Position % CHUNK_SIZE != 0)
{
stream.BaseStream.Position += CHUNK_SIZE - ((stream.BaseStream.Position - start) % CHUNK_SIZE);
}
}
// skip header block B
stream.ReadBytes((int)Header.NumEntriesB * 32);
// layout table entry block
for (int i = 0; i < Header.NumEntriesB; i++)
{
long start = stream.BaseStream.Position;
MD5Hash hash;
while (!(hash = new MD5Hash(stream)).IsEmpty)
{
var entry = new EncodingLayout()
{
Hash = hash,
StringIndex = stream.ReadUInt32BE(),
Size = stream.ReadUInt40BE()
};
Layout.Add(entry.Hash, entry);
}
if (stream.BaseStream.Position % CHUNK_SIZE != 0)
{
stream.BaseStream.Position += CHUNK_SIZE - ((stream.BaseStream.Position - start) % CHUNK_SIZE);
}
}
stream.ReadBytes((int)(stream.BaseStream.Length - stream.BaseStream.Position)); //EncodingStringTable
EncodingMap = blte.EncodingMap.ToArray();
blte?.Dispose();
stream?.Dispose();
}
private MD5Hash DownloadSystemFile(MD5Hash key, CDNClient client, string directory, string dataFolder = "data")
{
if (key.Value == null)
{
return(default);
public bool GetEntry(MD5Hash md5, out EncodingEntry enc) => EncodingData.TryGetValue(md5, out enc);
private Stream GetLocalDataStreamInternal(IndexEntry idxInfo, MD5Hash key)
{
if (idxInfo == null)
throw new Exception("Missing local index.");
Stream dataStream = GetDataStream(idxInfo.Index);
dataStream.Position = idxInfo.Offset;
using (BinaryReader reader = new BinaryReader(dataStream, System.Text.Encoding.ASCII, true))
{
byte[] md5 = reader.ReadBytes(16);
Array.Reverse(md5);
if (!key.EqualsTo(md5))
throw new Exception("local data corrupted");
int size = reader.ReadInt32();
if (size != idxInfo.Size)
throw new Exception("local data corrupted");
//byte[] unkData1 = reader.ReadBytes(2);
//byte[] unkData2 = reader.ReadBytes(8);
dataStream.Position += 10;
byte[] data = reader.ReadBytes(idxInfo.Size - 30);
return new MemoryStream(data);
}
}
public EncodingHandler(BinaryReader stream, BackgroundWorkerEx worker)
{
worker?.ReportProgress(0, "Loading \"encoding\"...");
stream.Skip(2); // EN
byte b1 = stream.ReadByte();
byte checksumSizeA = stream.ReadByte();
byte checksumSizeB = stream.ReadByte();
ushort flagsA = stream.ReadUInt16();
ushort flagsB = stream.ReadUInt16();
int numEntriesA = stream.ReadInt32BE();
int numEntriesB = stream.ReadInt32BE();
byte b4 = stream.ReadByte();
int stringBlockSize = stream.ReadInt32BE();
stream.Skip(stringBlockSize);
//string[] strings = Encoding.ASCII.GetString(stream.ReadBytes(stringBlockSize)).Split(new[] { '\0' }, StringSplitOptions.RemoveEmptyEntries);
stream.Skip(numEntriesA * 32);
//for (int i = 0; i < numEntriesA; ++i)
//{
// byte[] firstHash = stream.ReadBytes(16);
// byte[] blockHash = stream.ReadBytes(16);
//}
long chunkStart = stream.BaseStream.Position;
for (int i = 0; i < numEntriesA; ++i)
{
ushort keysCount;
while ((keysCount = stream.ReadUInt16()) != 0)
{
int fileSize = stream.ReadInt32BE();
MD5Hash md5 = stream.Read <MD5Hash>();
EncodingEntry entry = new EncodingEntry()
{
Size = fileSize
};
// how do we handle multiple keys?
for (int ki = 0; ki < keysCount; ++ki)
{
MD5Hash key = stream.Read <MD5Hash>();
// use first key for now
if (ki == 0)
{
entry.Key = key;
}
else
{
Logger.WriteLine("Multiple encoding keys for MD5 {0}: {1}", md5.ToHexString(), key.ToHexString());
}
}
//Encodings[md5] = entry;
EncodingData.Add(md5, entry);
}
// each chunk is 4096 bytes, and zero padding at the end
long remaining = CHUNK_SIZE - ((stream.BaseStream.Position - chunkStart) % CHUNK_SIZE);
if (remaining > 0)
{
stream.BaseStream.Position += remaining;
}
worker?.ReportProgress((int)((i + 1) / (float)numEntriesA * 100));
}
stream.Skip(numEntriesB * 32);
//for (int i = 0; i < numEntriesB; ++i)
//{
// byte[] firstKey = stream.ReadBytes(16);
// byte[] blockHash = stream.ReadBytes(16);
//}
long chunkStart2 = stream.BaseStream.Position;
for (int i = 0; i < numEntriesB; ++i)
{
byte[] key = stream.ReadBytes(16);
int stringIndex = stream.ReadInt32BE();
byte unk1 = stream.ReadByte();
int fileSize = stream.ReadInt32BE();
// each chunk is 4096 bytes, and zero padding at the end
long remaining = CHUNK_SIZE - ((stream.BaseStream.Position - chunkStart2) % CHUNK_SIZE);
if (remaining > 0)
{
stream.BaseStream.Position += remaining;
}
}
// string block till the end of file
}
public Stream OpenFile(MD5Hash hash)
{
try
{
return Program.Settings.UseRemote ? OpenFileOnline(hash) : OpenFileLocal(hash);
}
catch (Exception e) when (!(e is BLTEDecoderException))
{
return OpenFileOnline(hash);
}
}
private void ExtractFileLocal(MD5Hash key, string path, string name)
{
Stream stream = GetLocalDataStream(key);
using (BLTEStream blte = new BLTEStream(stream, key))
blte.ExtractToFile(path, name);
}
public void Read(BinaryReader br, InstallHeader header)
{
FilePath = br.ReadCString();
CKey = new MD5Hash(br.ReadBytes(header.CKeySize));
DecompressedSize = br.ReadUInt32BE();
}
public BLTEHandler(Stream stream, MD5Hash md5)
{
_reader = new BinaryReader(stream, Encoding.ASCII, true);
Parse(md5);
}
public static int UpdatePassWordByUserID(int userID, string password)
{
return(supplierUserDAL.UpdatePassWordByUserID(userID, MD5Hash.GetMd5String(password)));
}
private string ComputeHash()
{
return(MD5Hash.CreateHash(_fileLines.Value));
}
public static bool UpdatePassWordByToken(string token, string password)
{
return(supplierUserDAL.UpdatePasswordByToken(token, MD5Hash.GetMd5String(password)) > 0);
}
/// <summary>
/// Loads an existing DownloadFile
/// </summary>
/// <param name="directory">Base directory</param>
/// <param name="ekey">DownloadFile MD5</param>
public DownloadFile(string directory, MD5Hash ekey) :
this(Helpers.GetCDNPath(ekey.ToString(), "data", directory))
{
}
public TokenDTO Login(LoginDTO Usuario)
{
TokenDTO Retorno = new TokenDTO();
if (Usuario.Operador)
{
int Matricula;
try
{
Matricula = Convert.ToInt32(Usuario.Login);
}
catch
{
throw new Exception($"O login informado: {Usuario.Login} não é uma matrícula de operador válida.");
}
Operadores Operador = this._operadoresRepositorio.GetByLogin(Matricula);
if (Operador == null)
{
throw new Exception($"O login informado: {Usuario.Login} não existe.");
}
bool ValidaSenha = Operador.Senha.Equals(MD5Hash.GerarHashMd5(Usuario.Senha));
if (!ValidaSenha)
{
throw new Exception($"A senha para login informado: {Usuario.Login} não corresponde.");
}
Tuple <string, DateTime> Token = GerarToken(new UsuarioDTO {
Nome = Operador.Nome, Perfil = "Operador"
});
Retorno.Usuario = Operador.Nome;
Retorno.Perfil = "Operador";
Retorno.Token = Token.Item1;
Retorno.Expiracao = Token.Item2;
}
else
{
if (!ValidaCPF.ValidarCPF(Usuario.Login))
{
throw new Exception($"O login informado: {Usuario.Login} não é um CPF de cliente válido.");
}
Clientes Cliente = this._clientesRepositorio.GetByCpf(Usuario.Login);
if (Cliente == null)
{
throw new Exception($"O cliente informado: {Usuario.Login} não existe.");
}
bool ValidaSenha = Cliente.Senha.Equals(MD5Hash.GerarHashMd5(Usuario.Senha));
if (!ValidaSenha)
{
throw new Exception($"A senha para o cliente informado: {Usuario.Login} não corresponde.");
}
Tuple <string, DateTime> Token = GerarToken(new UsuarioDTO {
Nome = Cliente.Nome, Perfil = "Cliente"
});
Retorno.Usuario = Cliente.Nome;
Retorno.Perfil = "Cliente";
Retorno.Token = Token.Item1;
Retorno.Expiracao = Token.Item2;
}
return(Retorno);
}
protected override void ExtractFileOnline(MD5Hash key, string path, string name)
{
IndexEntry idxInfo = CDNIndex.GetIndexInfo(key);
if (idxInfo == null)
CDNIndexData.TryGetValue(key, out idxInfo);
ExtractFileOnlineInternal(idxInfo, key, path, name);
}
private void Parse(MD5Hash md5)
{
int size = (int)_reader.BaseStream.Length;
if (size < 8)
{
throw new BLTEDecoderException("not enough data: {0}", 8);
}
int magic = _reader.ReadInt32();
if (magic != BLTE_MAGIC)
{
throw new BLTEDecoderException("frame header mismatch (bad BLTE file)");
}
int headerSize = _reader.ReadInt32BE();
if (CASCConfig.ValidateData)
{
long oldPos = _reader.BaseStream.Position;
_reader.BaseStream.Position = 0;
byte[] newHash = _md5.ComputeHash(_reader.ReadBytes(headerSize > 0 ? headerSize : size));
if (!md5.EqualsTo(newHash))
{
throw new BLTEDecoderException("data corrupted");
}
_reader.BaseStream.Position = oldPos;
}
int numBlocks = 1;
if (headerSize > 0)
{
if (size < 12)
{
throw new BLTEDecoderException("not enough data: {0}", 12);
}
byte[] fcbytes = _reader.ReadBytes(4);
numBlocks = fcbytes[1] << 16 | fcbytes[2] << 8 | fcbytes[3] << 0;
if (fcbytes[0] != 0x0F || numBlocks == 0)
{
throw new BLTEDecoderException("bad table format 0x{0:x2}, numBlocks {1}", fcbytes[0], numBlocks);
}
int frameHeaderSize = 24 * numBlocks + 12;
if (headerSize != frameHeaderSize)
{
throw new BLTEDecoderException("header size mismatch");
}
if (size < frameHeaderSize)
{
throw new BLTEDecoderException("not enough data: {0}", frameHeaderSize);
}
}
DataBlock[] blocks = new DataBlock[numBlocks];
for (int i = 0; i < numBlocks; i++)
{
DataBlock block = new DataBlock();
if (headerSize != 0)
{
block.CompSize = _reader.ReadInt32BE();
block.DecompSize = _reader.ReadInt32BE();
block.Hash = _reader.Read <MD5Hash>();
}
else
{
block.CompSize = size - 8;
block.DecompSize = size - 8 - 1;
block.Hash = default(MD5Hash);
}
blocks[i] = block;
}
_memStream = new MemoryStream(blocks.Sum(b => b.DecompSize));
for (int i = 0; i < blocks.Length; i++)
{
DataBlock block = blocks[i];
block.Data = _reader.ReadBytes(block.CompSize);
if (!block.Hash.IsZeroed() && CASCConfig.ValidateData)
{
byte[] blockHash = _md5.ComputeHash(block.Data);
if (!block.Hash.EqualsTo(blockHash))
{
throw new BLTEDecoderException("MD5 mismatch");
}
}
HandleDataBlock(block.Data, i);
}
}
public Stream OpenDataFileDirect(MD5Hash key)
{
var keyStr = key.ToHexString().ToLower();
worker?.ReportProgress(0, string.Format("Downloading \"{0}\" file...", keyStr));
string file = config.CDNPath + "/data/" + keyStr.Substring(0, 2) + "/" + keyStr.Substring(2, 2) + "/" + keyStr;
string url = "http://" + config.CDNHost + "/" + file;
Stream stream = Cache.OpenFile(file, url, false);
if (stream != null)
return stream;
return downloader.OpenFile(url);
}
protected override Stream OpenFileOnline(MD5Hash key)
{
IndexEntry idxInfo = CDNIndex.GetIndexInfo(key);
return(OpenFileOnlineInternal(idxInfo, key));
}
private Stream OpenFileOnlineInternal(IndexEntry entry, MD5Hash hash)
{
Stream stream = entry != null ? CDNIndex.OpenDataFile(entry) : CDNIndex.OpenDataFileDirect(hash);
return new BLTEStream(stream, hash);
}
protected override Stream GetLocalDataStream(MD5Hash key)
{
IndexEntry idxInfo = LocalIndex.GetIndexInfo(key);
return(GetLocalDataStreamInternal(idxInfo, key));
}
public Stream OpenFileOnline(MD5Hash hash)
{
IndexEntry entry = CDNIndex.GetIndexInfo(hash);
return OpenFileOnlineInternal(entry, hash);
}
protected override void ExtractFileOnline(MD5Hash key, string path, string name)
{
IndexEntry idxInfo = CDNIndex.GetIndexInfo(key);
ExtractFileOnlineInternal(idxInfo, key, path, name);
}
private void Parse(MD5Hash hash)
{
int size = (int)reader.BaseStream.Length;
if (size < 8)
throw new BLTEDecoderException("Invalid data length: {0}", 8);
int magic = reader.ReadInt32();
if (magic != BLTE_MAGIC)
throw new BLTEDecoderException("Mis-matched header (magic)");
int headerSize = reader.ReadInt32BE();
if (CASCConfig.ValidateData)
{
long oldPos = reader.BaseStream.Position;
reader.BaseStream.Position = 0;
byte[] newHash = this.hash.ComputeHash(reader.ReadBytes(headerSize > 0 ? headerSize : size));
if (!hash.EqualsTo(newHash))
throw new BLTEDecoderException("Corrupted data?");
reader.BaseStream.Position = oldPos;
}
int numBlocks = 1;
if (headerSize > 0)
{
if (size < 12)
throw new BLTEDecoderException("Not enough data: {0}", 12);
byte[] bytes = reader.ReadBytes(4);
numBlocks = bytes[1] << 16 | bytes[2] << 8 | bytes[3] << 0;
if (bytes[0] != 0x0F || numBlocks == 0)
throw new BLTEDecoderException("Invalid table format 0x{0:x2}, numBlocks {1}", bytes[0], numBlocks);
int frameHeaderSize = 24 * numBlocks + 12;
if (headerSize != frameHeaderSize)
throw new BLTEDecoderException("Header size mis-match");
if (size < frameHeaderSize)
throw new BLTEDecoderException("Not enough data: {0}", frameHeaderSize);
}
dataBlocks = new DataBlock[numBlocks];
for (int i = 0; i < numBlocks; i++)
{
DataBlock block = new DataBlock();
if (headerSize != 0)
{
block.CompSize = reader.ReadInt32BE();
block.DecompSize = reader.ReadInt32BE();
block.Hash = reader.Read<MD5Hash>();
}
else
{
block.CompSize = size - 8;
block.DecompSize = size - 8 - 1;
block.Hash = default(MD5Hash);
}
dataBlocks[i] = block;
}
streamMemory = new MemoryStream(dataBlocks.Sum(b => b.DecompSize));
ProcessNextBlock();
length = headerSize == 0 ? streamMemory.Length : streamMemory.Capacity;
}
/// <summary>Create a MD5FileChecksum</summary>
public MD5MD5CRC32FileChecksum(int bytesPerCRC, long crcPerBlock, MD5Hash md5)
{
this.bytesPerCRC = bytesPerCRC;
this.crcPerBlock = crcPerBlock;
this.md5 = md5;
}
private void btnCadastrar_Click(object sender, EventArgs e)
{
try
{
string confirmarSenha = "";
u.nome = txtNome.Text;
u.login = txtLogin.Text;
u.senha = txtSenha.Text;
confirmarSenha = txtConfirmarSenha.Text;
string q = "select login_usuario from tcc.usuarios where login_usuario='" + u.login + "'";
string w = "select email_usuario from tcc.usuarios where email_usuario='" + u.email + "'";
c.abrirConexao();
MySqlCommand verificaLogin = new MySqlCommand(q, c.mcon);
MySqlDataReader readerLogin = verificaLogin.ExecuteReader();
if (readerLogin.Read())
{
if (u.login == "")
{
MessageBox.Show("Nenhum campo pode ficar em branco!", "Aviso", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
c.fecharConexao();
}
else
{
MessageBox.Show("Esse login já está em uso, por favor utilize outro!", "Aviso", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
readerLogin.Close();
c.fecharConexao();
}
}
else
{
u.tipoUsuario = 1;
u.status = 0;
u.reset = 0;
if (u.nome == "" || u.login == "" || u.senha == "" || confirmarSenha == "")
{
MessageBox.Show("Nenhum campo pode ficar em branco!", "Aviso", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
c.fecharConexao();
}
else if (u.senha != confirmarSenha)
{
MessageBox.Show("As senhas não conferem!", "Aviso", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
c.fecharConexao();
}
else if (validaSenha >= 1)
{
MessageBox.Show("Melhore a força da senha!", "Aviso", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
c.fecharConexao();
}
else
{
u.senhacrip = MD5Hash.CalculaHash(u.senha);
u.cadastrarUsuario(u.tipoUsuario);
this.Close();
}
}
}
catch (Exception ex)
{
MessageBox.Show("Erro ao cadastrar o instrutor! \n" + ex, "Erro", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
/// <exception cref="System.IO.IOException"/>
public override void ReadFields(BinaryReader reader)
{
bytesPerCRC = @in.ReadInt();
crcPerBlock = @in.ReadLong();
md5 = MD5Hash.Read(@in);
}
private void CheckThread()
{
try
{
signatures = new List <Signature> ();
string sroot = path + "Scenario/";
string wroot = path + "Worlds/";
string mroot = sroot + "Missions/";
string fpath = sroot + "ForceSettings.xml";
string wpath = sroot + "WorldSettings.xml";
// World settings
if (!AddFileSig("_WS", wpath))
{
integrated = false;
error = "WorldSettings.xml corrupt";
return;
}
// Force settings
if (!AddFileSig("_FS", fpath))
{
integrated = false;
error = "ForceSettings.xml corrupt";
return;
}
// Missions
if (Directory.Exists(mroot))
{
string[] mfs = Directory.GetFiles(mroot);
foreach (string mf in mfs)
{
FileInfo fi = new FileInfo(mf);
int id = 0;
if (fi.Name.Length < 5)
{
integrated = false;
error = "Corrupt mission filename (5)";
return;
}
string idstr = fi.Name.Substring(0, fi.Name.Length - 4);
if (int.TryParse(idstr, out id))
{
if (!AddFileSig("M" + idstr, mf))
{
integrated = false;
error = "Mission " + idstr + " corrupt";
return;
}
}
else
{
integrated = false;
error = "Corrupt mission filename (x)";
return;
}
}
}
else
{
integrated = false;
error = "Mission folder lost";
return;
}
// Worlds
if (Directory.Exists(wroot))
{
XmlDocument doc = new XmlDocument();
doc.Load(wpath);
List <string> worldpaths = new List <string> ();
foreach (XmlNode node in doc.DocumentElement.ChildNodes)
{
if (node.Name == "WORLD")
{
worldpaths.Add(node.Attributes["path"].Value);
}
else
{
integrated = false;
error = "WorldSettings.xml corrupt";
return;
}
}
for (int w = 0; w < worldpaths.Count; ++w)
{
string worldpath = wroot + worldpaths[w] + "/";
for (int i = 0; i < worldFiles.Length; ++i)
{
string filepath = worldpath + worldFiles[i];
AddFileSig("W" + w.ToString() + worldFileCodes[i], filepath);
}
}
}
else
{
integrated = false;
error = "World folder lost";
return;
}
// Hash
signatures.Sort(Signature.Compare);
string s = "";
for (int i = 0; i < signatures.Count; ++i)
{
s += signatures[i].code;
s += ":";
s += signatures[i].sig;
s += ";";
}
string tstuid = MD5Hash.MD5Encoding(s + sp);
if (uid == tstuid)
{
integrated = true;
return;
}
else
{
integrated = false;
error = "One or more file has been tampered";
return;
}
}
catch (Exception e)
{
error = "TryCatch " + e.ToString();
integrated = false;
return;
}
}
/// <summary>
/// Updates the Url and the Url Data tables
/// </summary>
/// <param name="data">The UrlCrawlData containing the data of the crawled Url.</param>
/// <param name="transaction">The currently active <see cref="SqlTransaction"/>.</param>
/// <returns>The ID of the updated url or 0 of something goes wrong.</returns>
private int UpdateUrl(UrlCrawlData data, SqlTransaction transaction)
{
int retVal = 0;
try
{
//build the Sql Command for updating the url table
SqlCommand urlcmd = new SqlCommand("cw_update_url", dbcon, transaction);
urlcmd.CommandType = CommandType.StoredProcedure;
urlcmd.CommandTimeout = settings.DBActionTimeout;
urlcmd.Parameters.Add("@url_id", SqlDbType.Int);
urlcmd.Parameters.Add("@url", SqlDbType.NVarChar, 500);
urlcmd.Parameters.Add("@url_md5", SqlDbType.UniqueIdentifier);
urlcmd.Parameters.Add("@url_host_id", SqlDbType.UniqueIdentifier);
urlcmd.Parameters.Add("@url_priority", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@crc", SqlDbType.BigInt);
urlcmd.Parameters.Add("@flag_domain", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@flag_robots", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@flag_updated", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@last_visited", SqlDbType.SmallDateTime);
urlcmd.Parameters.Add("@flag_redirected", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@id", SqlDbType.Int);
urlcmd.Parameters["@id"].Direction = ParameterDirection.Output;
//Build the SQL Command for updating the hosts table
SqlCommand hostcmd = new SqlCommand("cw_insert_host", dbcon, transaction);
hostcmd.CommandType = CommandType.StoredProcedure;
hostcmd.CommandTimeout = settings.DBActionTimeout;
hostcmd.Parameters.Add("@host_id", SqlDbType.UniqueIdentifier);
hostcmd.Parameters.Add("@host_name", SqlDbType.NVarChar, 100);
//set their parameters
urlcmd.Parameters[0].Value = data.ID;
urlcmd.Parameters[1].Value = data.Url;
urlcmd.Parameters[2].Value = new Guid(data.MD5);
Uri uri = new Uri(data.Url);
string host_name = uri.Host;
Guid host_id = new Guid(MD5Hash.md5(host_name));
urlcmd.Parameters[3].Value = host_id;
urlcmd.Parameters[5].Value = data.CRC;
if (data.Redirected)
{
//we must first attempt to insert the host, otherwise the urlcmd will fail
hostcmd.Parameters[0].Value = host_id;
hostcmd.Parameters[1].Value = host_name;
try
{
hostcmd.ExecuteNonQuery();
}
catch
{
//it probably exists already
}
urlcmd.Parameters[4].Value = (byte)data.RedirectedPriority;
urlcmd.Parameters[6].Value = (byte)data.RedirectedFlagDomain;
urlcmd.Parameters[7].Value = (data.RedirectedFlagRobots)?1:0;
urlcmd.Parameters[8].Value = (data.Updated)?1:0;
urlcmd.Parameters[9].Value = data.TimeStamp;
urlcmd.Parameters[10].Value = 1;
}
else
{
urlcmd.Parameters[4].Value = DBNull.Value;
urlcmd.Parameters[6].Value = (byte)data.UrlToCrawl.FlagDomain;
if (data.FlagFetchRobots)
{
urlcmd.Parameters[7].Value = (data.RedirectedFlagRobots)?1:0;
}
else
{
urlcmd.Parameters[7].Value = 0;
}
urlcmd.Parameters[8].Value = (data.Updated)?1:0;
urlcmd.Parameters[9].Value = data.TimeStamp;
urlcmd.Parameters[10].Value = 0;
}
//retVal = data.ID;
//make sure the host command is disposed
hostcmd.Dispose();
urlcmd.ExecuteNonQuery();
retVal = (int)urlcmd.Parameters["@id"].Value;
urlcmd.Dispose();
if (data.Updated)
{
//if necessary build the sql command for updating the url data tables
SqlCommand urldatacmd = new SqlCommand("cw_update_url_data", dbcon, transaction);
urldatacmd.CommandType = CommandType.StoredProcedure;
urldatacmd.CommandTimeout = settings.DBActionTimeout;
urldatacmd.Parameters.Add("@url_id", SqlDbType.Int);
urldatacmd.Parameters.Add("@data", SqlDbType.Image);
urldatacmd.Parameters.Add("@length", SqlDbType.Int);
urldatacmd.Parameters.Add("@original_length", SqlDbType.Int);
urldatacmd.Parameters.Add("@http_code", SqlDbType.SmallInt);
urldatacmd.Parameters.Add("@retrieval_time", SqlDbType.Int);
urldatacmd.Parameters[0].Value = retVal;
//compress the url's data
if (data.Data != String.Empty)
{
byte [] compressed = null;
string urldata = InternetUtils.Base64Decode(data.Data);
CompressionUtils.CompressString(ref urldata, out compressed);
urldatacmd.Parameters[1].Value = compressed;
urldatacmd.Parameters[2].Value = compressed.Length;
urldatacmd.Parameters[3].Value = data.Data.Length;
}
else
{
urldatacmd.Parameters[1].Value = new byte[0];
urldatacmd.Parameters[2].Value = 0;
urldatacmd.Parameters[3].Value = 0;
}
urldatacmd.Parameters[4].Value = (short)data.HttpStatusCode;
urldatacmd.Parameters[5].Value = data.RetrievalTime;
urldatacmd.ExecuteNonQuery();
urldatacmd.Dispose();
}
}
catch (Exception e)
{
AddToReportQueue(CWLoggerEntryType.Warning, "DBUpdater failed to update a Url in the database: " + e.ToString());
retVal = 0;
}
return(retVal);
}
public EncodingHandler(BinaryReader stream)
{
stream.Skip(2); // EN
byte Version = stream.ReadByte(); // must be 1
byte CKeyLength = stream.ReadByte();
byte EKeyLength = stream.ReadByte();
int CKeyPageSize = stream.ReadInt16BE() * 1024; // KB to bytes
int EKeyPageSize = stream.ReadInt16BE() * 1024; // KB to bytes
int CKeyPageCount = stream.ReadInt32BE();
int EKeyPageCount = stream.ReadInt32BE();
byte unk1 = stream.ReadByte(); // must be 0
int ESpecBlockSize = stream.ReadInt32BE();
stream.Skip(ESpecBlockSize);
//string[] strings = Encoding.ASCII.GetString(stream.ReadBytes(ESpecBlockSize)).Split(new[] { '\0' }, StringSplitOptions.RemoveEmptyEntries);
stream.Skip(CKeyPageCount * 32);
//ValueTuple<byte[], byte[]>[] aEntries = new ValueTuple<byte[], byte[]>[CKeyPageCount];
//for (int i = 0; i < CKeyPageCount; ++i)
//{
// byte[] firstHash = stream.ReadBytes(16);
// byte[] blockHash = stream.ReadBytes(16);
// aEntries[i] = (firstHash, blockHash);
//}
long chunkStart = stream.BaseStream.Position;
for (int i = 0; i < CKeyPageCount; ++i)
{
byte keysCount;
while ((keysCount = stream.ReadByte()) != 0)
{
long fileSize = stream.ReadInt40BE();
MD5Hash cKey = stream.Read <MD5Hash>();
EncodingEntry entry = new EncodingEntry()
{
Size = fileSize
};
// how do we handle multiple keys?
for (int ki = 0; ki < keysCount; ++ki)
{
MD5Hash eKey = stream.Read <MD5Hash>();
// use first key for now
if (ki == 0)
{
entry.Key = eKey;
}
//else
// Logger.WriteLine("Multiple encoding keys for MD5 {0}: {1}", md5.ToHexString(), key.ToHexString());
//Logger.WriteLine("Encoding {0:D2} {1} {2} {3} {4}", keysCount, aEntries[i].Item1.ToHexString(), aEntries[i].Item2.ToHexString(), md5.ToHexString(), key.ToHexString());
}
//Encodings[md5] = entry;
EncodingData.Add(cKey, entry);
}
// each chunk is 4096 bytes, and zero padding at the end
long remaining = CHUNK_SIZE - ((stream.BaseStream.Position - chunkStart) % CHUNK_SIZE);
if (remaining > 0)
{
stream.BaseStream.Position += remaining;
}
}
stream.Skip(EKeyPageCount * 32);
//for (int i = 0; i < EKeyPageCount; ++i)
//{
// byte[] firstKey = stream.ReadBytes(16);
// byte[] blockHash = stream.ReadBytes(16);
//}
long chunkStart2 = stream.BaseStream.Position;
for (int i = 0; i < EKeyPageCount; ++i)
{
byte[] eKey = stream.ReadBytes(16);
int eSpecIndex = stream.ReadInt32BE();
long fileSize = stream.ReadInt40BE();
// each chunk is 4096 bytes, and zero padding at the end
long remaining = CHUNK_SIZE - ((stream.BaseStream.Position - chunkStart2) % CHUNK_SIZE);
if (remaining > 0)
{
stream.BaseStream.Position += remaining;
}
}
// string block till the end of file
//EncodingData.Dump();
}
/// <summary>
/// Inserts the links contained in a url into the database and updates the link graph
/// </summary>
/// <param name="UrlID">The ID of the url.</param>
/// <param name="data">The <see cref="UrlCrawlData"/> of the url.</param>
/// <param name="transaction">The currently active <see cref="SqlTransaction"/>.</param>
private void InsertUrlOutLinks(int UrlID, UrlCrawlData data, SqlTransaction transaction)
{
try
{
//Build the SQL Commands
SqlCommand hostcmd = new SqlCommand("cw_insert_host", dbcon, transaction);
hostcmd.CommandType = CommandType.StoredProcedure;
hostcmd.CommandTimeout = settings.DBActionTimeout;
hostcmd.Parameters.Add("@host_id", SqlDbType.UniqueIdentifier);
hostcmd.Parameters.Add("@host_name", SqlDbType.NVarChar, 100);
SqlCommand urlcmd = new SqlCommand("cw_insert_url", dbcon, transaction);
urlcmd.CommandType = CommandType.StoredProcedure;
urlcmd.CommandTimeout = settings.DBActionTimeout;
urlcmd.Parameters.Add("@url", SqlDbType.NVarChar, 500);
urlcmd.Parameters.Add("@url_md5", SqlDbType.UniqueIdentifier);
urlcmd.Parameters.Add("@url_host_id", SqlDbType.UniqueIdentifier);
urlcmd.Parameters.Add("@url_priority", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@flag_domain", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@flag_robots", SqlDbType.TinyInt);
urlcmd.Parameters.Add("@id", SqlDbType.Int);
urlcmd.Parameters["@id"].Direction = ParameterDirection.Output;
SqlCommand linkcmd = new SqlCommand("cw_insert_link_graph", dbcon, transaction);
linkcmd.CommandType = CommandType.StoredProcedure;
linkcmd.CommandTimeout = settings.DBActionTimeout;
linkcmd.Parameters.Add("@from_url_id", SqlDbType.Int);
linkcmd.Parameters.Add("@to_url_id", SqlDbType.Int);
int new_id = 0;
//insert each out link in the database
foreach (InternetUrlToIndex url in data.OutLinks)
{
try
{
Uri uri = new Uri(url.Url);
Guid host_id = new Guid(MD5Hash.md5(uri.Host));
hostcmd.Parameters[0].Value = host_id;
hostcmd.Parameters[1].Value = uri.Host;
hostcmd.ExecuteNonQuery();
urlcmd.Parameters[0].Value = url.Url;
urlcmd.Parameters[1].Value = new Guid(url.MD5);
urlcmd.Parameters[2].Value = host_id;
urlcmd.Parameters[3].Value = (byte)url.Priority;
urlcmd.Parameters[4].Value = (byte)url.FlagDomain;
urlcmd.Parameters[5].Value = (byte)((url.FlagRobots)?1:0);
urlcmd.ExecuteNonQuery();
new_id = (int)urlcmd.Parameters["@id"].Value; //(int)urlcmd.ExecuteScalar();
linkcmd.Parameters[0].Value = UrlID;
linkcmd.Parameters[1].Value = new_id;
linkcmd.ExecuteNonQuery();
}
catch (Exception e)
{
AddToReportQueue(CWLoggerEntryType.Warning, "DBUpdater plugin failed to insert an edge to the link graph: " + e.ToString());
continue;
}
}
urlcmd.Dispose();
linkcmd.Dispose();
}
catch (Exception e)
{
AddToReportQueue(CWLoggerEntryType.Warning, "DBUpdater plugin: an unexpected error occured when inserting the out links of url with ID " + UrlID.ToString() + " to the link graph: " + e.ToString());
GC.Collect();
}
}
protected override Stream GetLocalDataStream(MD5Hash key)
{
IndexEntry idxInfo;
if (LocalIndex != null)
idxInfo = LocalIndex.GetIndexInfo(key);
else
LocalIndexData.TryGetValue(key, out idxInfo);
return GetLocalDataStreamInternal(idxInfo, key);
}
public void SaveToFile(string filename)
{
var tmpIndexMap = string.Format("{0}.{1}.indexmap.tmp", filename, Guid.NewGuid());
using (var memStream = new MemoryStream())
using (var memWriter = new StreamWriter(memStream))
{
memWriter.WriteLine(new string('0', 32)); // pre-allocate space for MD5 hash
memWriter.WriteLine(Version);
memWriter.WriteLine("{0}/{1}", PrepareCheckpoint, CommitCheckpoint);
memWriter.WriteLine(_maxTableLevelsForAutomaticMerge);
for (int i = 0; i < _map.Count; i++)
{
for (int j = 0; j < _map[i].Count; j++)
{
memWriter.WriteLine("{0},{1},{2}", i, j, new FileInfo(_map[i][j].Filename).Name);
}
}
memWriter.Flush();
memStream.Position = 32;
var hash = MD5Hash.GetHashFor(memStream);
memStream.Position = 0;
foreach (var t in hash)
{
memWriter.Write(t.ToString("X2"));
}
memWriter.Flush();
memStream.Position = 0;
using (var f = File.OpenWrite(tmpIndexMap))
{
f.Write(memStream.GetBuffer(), 0, (int)memStream.Length);
f.FlushToDisk();
}
}
int trial = 0;
int maxTrials = 5;
while (trial < maxTrials)
{
Action <Exception> errorHandler = ex =>
{
Log.Error("Failed trial to replace indexmap {indexMap} with {tmpIndexMap}.", filename, tmpIndexMap);
Log.Error("Exception: {e}", ex);
trial += 1;
};
try
{
if (File.Exists(filename))
{
File.SetAttributes(filename, FileAttributes.Normal);
File.Delete(filename);
}
File.Move(tmpIndexMap, filename);
break;
}
catch (IOException exc)
{
errorHandler(exc);
if (trial >= maxTrials)
{
ProcessUtil.PrintWhoIsLocking(tmpIndexMap, Log);
ProcessUtil.PrintWhoIsLocking(filename, Log);
}
}
catch (UnauthorizedAccessException exc)
{
errorHandler(exc);
}
}
}
protected override Stream OpenFileOnline(MD5Hash key)
{
IndexEntry idxInfo = CDNIndex.GetIndexInfo(key);
if (idxInfo == null)
CDNIndexData.TryGetValue(key, out idxInfo);
return OpenFileOnlineInternal(idxInfo, key);
}
public D3RootHandler(BinaryReader stream, BackgroundWorkerEx worker, CASCHandler casc)
{
worker?.ReportProgress(0, "Loading \"root\"...");
byte b1 = stream.ReadByte();
byte b2 = stream.ReadByte();
byte b3 = stream.ReadByte();
byte b4 = stream.ReadByte();
int count = stream.ReadInt32();
for (int j = 0; j < count; j++)
{
MD5Hash md5 = stream.Read <MD5Hash>();
string name = stream.ReadCString();
var entries = new List <D3RootEntry>();
D3RootData[name] = entries;
if (!casc.Encoding.GetEntry(md5, out EncodingEntry enc))
{
continue;
}
using (BinaryReader s = new BinaryReader(casc.OpenFile(enc.Key)))
{
uint magic = s.ReadUInt32();
int nEntries0 = s.ReadInt32();
for (int i = 0; i < nEntries0; i++)
{
entries.Add(D3RootEntry.Read(0, s));
}
int nEntries1 = s.ReadInt32();
for (int i = 0; i < nEntries1; i++)
{
entries.Add(D3RootEntry.Read(1, s));
}
int nNamedEntries = s.ReadInt32();
for (int i = 0; i < nNamedEntries; i++)
{
entries.Add(D3RootEntry.Read(2, s));
}
}
worker?.ReportProgress((int)((j + 1) / (float)(count + 2) * 100));
}
// Parse CoreTOC.dat
var coreTocEntry = D3RootData["Base"].Find(e => e.Name == "CoreTOC.dat");
casc.Encoding.GetEntry(coreTocEntry.MD5, out EncodingEntry enc1);
using (var file = casc.OpenFile(enc1.Key))
tocParser = new CoreTOCParser(file);
worker?.ReportProgress((int)((count + 1) / (float)(count + 2) * 100));
// Parse Packages.dat
var pkgEntry = D3RootData["Base"].Find(e => e.Name == "Data_D3\\PC\\Misc\\Packages.dat");
casc.Encoding.GetEntry(pkgEntry.MD5, out EncodingEntry enc2);
using (var file = casc.OpenFile(enc2.Key))
pkgParser = new PackagesParser(file);
worker?.ReportProgress(100);
}
public void opEquality_MD5Hash_MD5Hash()
{
var obj = new MD5Hash();
var comparand = new MD5Hash();
Assert.True(obj == comparand);
}
private string GetDbPathHash(string dbPath)
{
using (var memStream = new MemoryStream(Helper.UTF8NoBom.GetBytes(dbPath))) {
return(BitConverter.ToString(MD5Hash.GetHashFor(memStream)).Replace("-", ""));
}
}
private Stream OpenFileLocal(MD5Hash key)
{
Stream stream = GetLocalDataStream(key);
return new BLTEStream(stream, key);
}
static void SaveBuildList(string outputPath)
{
AssetList buildList = new AssetList();
HashSet <string> bundeList = new HashSet <string>();
buildList.manifest = EditorUserBuildSettings.activeBuildTarget.ToString();
bundeList.Add(buildList.manifest);
foreach (var file in list.assets.Values)
{
if (file.asset.Contains("resources/") == false || string.IsNullOrEmpty(file.bundle) == false)
{
if (buildList.Contains(file.name) == false && buildList.assets.ContainsValue(file) == false)
{
if (string.IsNullOrEmpty(file.bundle))
{
string fullPath = string.Format("{0}{1}", outputPath, file.asset);
byte[] bytes = File.ReadAllBytes(fullPath);
file.size = bytes.Length;
file.md5 = MD5Hash.Get(bytes);
}
else
{
file.size = 0;
file.md5 = null;
bundeList.Add(file.bundle);
}
buildList.assets.Add(file.name, file);
}
}
else
{
if (buildList.Contains(file.name) == false && buildList.assets.ContainsValue(file) == false)
{
string fullPath = string.Format("{0}{1}", Application.dataPath.Replace("Assets", ""), file.asset);
byte[] bytes = File.ReadAllBytes(fullPath);
file.size = bytes.Length;
file.md5 = MD5Hash.Get(bytes);
buildList.assets.Add(file.name, file);
}
}
}
foreach (var name in bundeList)
{
string fullPath = string.Format("{0}{1}", outputPath, name);
byte[] bytes = File.ReadAllBytes(fullPath);
AssetFile file = new AssetFile();
file.name = name;
file.bundle = name;
file.size = bytes.Length;
file.md5 = MD5Hash.Get(bytes);
buildList.assets.Add(file.name, file);
}
string assetXmlFile = outputPath + "/" + AssetPath.ASSETSFILE;
StreamWriter writerXml = new StreamWriter(assetXmlFile);
writerXml.Write(buildList.ToXml());
writerXml.Close();
writerXml.Dispose();
}
public void SaveFileTo(MD5Hash key, string path, string name)
{
try
{
if (Program.Settings.UseRemote)
ExtractFileOnline(key, path, name);
else
ExtractFileLocal(key, path, name);
}
catch
{
ExtractFileOnline(key, path, name);
}
}
public bool GetEntry(MD5Hash hash, out EncodingEntry entry)
{
return EncodingData.TryGetValue(hash, out entry);
}
private void ExtractFileOnline(MD5Hash key, string path, string name)
{
IndexEntry entry = CDNIndex.GetIndexInfo(key);
if (entry != null)
{
using (Stream s = CDNIndex.OpenDataFile(entry))
using (BLTEStream blte = new BLTEStream(s, key))
{
blte.ExtractToFile(path, name);
}
}
else
{
using (Stream s = CDNIndex.OpenDataFileDirect(key))
using (BLTEStream blte = new BLTEStream(s, key))
{
blte.ExtractToFile(path, name);
}
}
}
protected override Stream GetLocalDataStream(MD5Hash key)
{
IndexEntry idxInfo = LocalIndex.GetIndexInfo(key);
if (idxInfo == null)
{
Logger.WriteLine("Local index missing: {0}", key.ToHexString());
}
return GetLocalDataStreamInternal(idxInfo, key);
}
public Stream GetLocalDataStream(MD5Hash hash)
{
IndexEntry entry = LocalIndex.GetIndexInfo(hash);
if (entry == null)
Log.Write("Missing local index: {0}", hash.ToHexString());
return GetLocalDataStreamInternal(entry, hash);
}
public Stream OpenDataFileDirect(MD5Hash key)
{
var keyStr = key.ToHexString().ToLower();
string file = Program.Settings.RemoteHostPath + "/data/" + keyStr.Substring(0, 2) + "/" + keyStr.Substring(2, 2) + "/" + keyStr;
string url = string.Format("http://{0}/{1}", Program.Settings.RemoteHost, file);
Stream stream = Cache.OpenFile(file, url, false);
if (stream != null)
return stream;
return downloader.OpenFile(url);
}
private BinaryReader OpenInstallationFile(EncodingHandler encoding, CASCEngine engine, MD5Hash hash, string errorName)
{
EncodingEntry entry;
if (!encoding.GetEntry(hash, out entry))
throw new FileNotFoundException(string.Format("Installation missing {0} file!", errorName));
return new BinaryReader(engine.OpenFile(entry.Key));
}
public DownloadEntry GetEntry(MD5Hash key)
{
DownloadEntry entry;
DownloadData.TryGetValue(key, out entry);
if (entry != null && entry.Tags == null)
entry.Tags = Tags.Where(kv => kv.Value.Bits[entry.Index]);
return entry;
}
/// <summary> /// Returns a PatchEntry by CKey /// </summary> /// <param name="ckey"></param> /// <param name="entry"></param> /// <returns></returns> public bool TryGet(MD5Hash ckey, out PatchEntry entry) => _PatchEntries.TryGetValue(ckey, out entry);
private void Parse(MD5Hash md5)
{
int size = (int)_reader.BaseStream.Length;
if (size < 8)
throw new BLTEDecoderException("not enough data: {0}", 8);
int magic = _reader.ReadInt32();
if (magic != BLTE_MAGIC)
throw new BLTEDecoderException("frame header mismatch (bad BLTE file)");
int headerSize = _reader.ReadInt32BE();
if (CASCConfig.ValidateData)
{
long oldPos = _reader.BaseStream.Position;
_reader.BaseStream.Position = 0;
byte[] newHash = _md5.ComputeHash(_reader.ReadBytes(headerSize > 0 ? headerSize : size));
if (!md5.EqualsTo(newHash))
throw new BLTEDecoderException("data corrupted");
_reader.BaseStream.Position = oldPos;
}
int numBlocks = 1;
if (headerSize > 0)
{
if (size < 12)
throw new BLTEDecoderException("not enough data: {0}", 12);
byte[] fcbytes = _reader.ReadBytes(4);
numBlocks = fcbytes[1] << 16 | fcbytes[2] << 8 | fcbytes[3] << 0;
if (fcbytes[0] != 0x0F || numBlocks == 0)
throw new BLTEDecoderException("bad table format 0x{0:x2}, numBlocks {1}", fcbytes[0], numBlocks);
int frameHeaderSize = 24 * numBlocks + 12;
if (headerSize != frameHeaderSize)
throw new BLTEDecoderException("header size mismatch");
if (size < frameHeaderSize)
throw new BLTEDecoderException("not enough data: {0}", frameHeaderSize);
}
_dataBlocks = new DataBlock[numBlocks];
for (int i = 0; i < numBlocks; i++)
{
DataBlock block = new DataBlock();
if (headerSize != 0)
{
block.CompSize = _reader.ReadInt32BE();
block.DecompSize = _reader.ReadInt32BE();
block.Hash = _reader.Read<MD5Hash>();
}
else
{
block.CompSize = size - 8;
block.DecompSize = size - 8 - 1;
block.Hash = default(MD5Hash);
}
_dataBlocks[i] = block;
}
_memStream = new MemoryStream(_dataBlocks.Sum(b => b.DecompSize));
ProcessNextBlock();
_length = headerSize == 0 ? _memStream.Length : _memStream.Capacity;
//for (int i = 0; i < _dataBlocks.Length; i++)
//{
// ProcessNextBlock();
//}
}
protected override Stream OpenFileOnline(MD5Hash key)
{
IndexEntry idxInfo = CDNIndex.GetIndexInfo(key);
return OpenFileOnlineInternal(idxInfo, key);
}
/// <summary> /// Determines whether the specific CKey exists /// </summary> /// <param name="ckey"></param> /// <returns></returns> public bool ContainsKey(MD5Hash ckey) => _PatchEntries.ContainsKey(ckey);
protected override void ExtractFileOnline(MD5Hash key, string path, string name)
{
IndexEntry idxInfo = CDNIndex.GetIndexInfo(key);
ExtractFileOnlineInternal(idxInfo, key, path, name);
}
/// <summary>
/// Loads an existing PatchFile
/// </summary>
/// <param name="directory">Base directory</param>
/// <param name="ekey">PatchFile MD5</param>
public PatchFile(string directory, MD5Hash ekey) :
this(Helpers.GetCDNPath(ekey.ToString(), "patch", directory))
{
}
public IndexEntry GetIndexInfo(MD5Hash key)
{
IndexEntry result;
if (!CDNIndexData.TryGetValue(key, out result))
Log.Write("CASC: CDNIndexHandler missing index {0}", key.ToHexString());
return result;
}
public static IndexMap FromFile(string filename, Func <IndexEntry, bool> isHashCollision, int maxTablesPerLevel = 4)
{
var tables = new List <List <PTable> >();
int version;
long prepareCheckpoint = -1;
long commitCheckpoint = -1;
if (!File.Exists(filename))
{
return(new IndexMap(IndexMapVersion, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel));
}
using (var f = File.OpenRead(filename))
using (var reader = new StreamReader(f))
{
// calculate real MD5 hash except first 32 bytes which are string representation of stored hash
f.Position = 32;
var realHash = MD5Hash.GetHashFor(f);
f.Position = 0;
// read stored MD5 hash and convert it from string to byte array
string text;
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("IndexMap file is empty.");
}
if (text.Length != 32 || !text.All(x => char.IsDigit(x) || (x >= 'A' && x <= 'F')))
{
throw new CorruptIndexException("Corrupted MD5 hash.");
}
// check expected and real hashes are the same
var expectedHash = new byte[16];
for (int i = 0; i < 16; ++i)
{
expectedHash[i] = Convert.ToByte(text.Substring(i * 2, 2), 16);
}
if (expectedHash.Length != realHash.Length)
{
throw new InvalidOperationException("Invalid length of expected and real hash.");
}
for (int i = 0; i < realHash.Length; ++i)
{
if (expectedHash[i] != realHash[i])
{
throw new CorruptIndexException("Expected and real hash are different.");
}
}
// at this point we can assume the format is ok, so actually no need to check errors.
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("Corrupted version.");
}
version = int.Parse(text);
// read and check prepare/commit checkpoint
if ((text = reader.ReadLine()) == null)
{
throw new CorruptIndexException("Corrupted commit checkpoint.");
}
try
{
var checkpoints = text.Split('/');
if (!long.TryParse(checkpoints[0], out prepareCheckpoint) || prepareCheckpoint < -1)
{
throw new CorruptIndexException("Invalid prepare checkpoint.");
}
if (!long.TryParse(checkpoints[1], out commitCheckpoint) || commitCheckpoint < -1)
{
throw new CorruptIndexException("Invalid commit checkpoint.");
}
}
catch (Exception exc)
{
throw new CorruptIndexException("Corrupted prepare/commit checkpoints pair.", exc);
}
// all next lines are PTables sorted by levels
while ((text = reader.ReadLine()) != null)
{
if (prepareCheckpoint < 0 || commitCheckpoint < 0)
{
throw new CorruptIndexException("Negative prepare/commit checkpoint in non-empty IndexMap.");
}
PTable ptable = null;
var pieces = text.Split(',');
try
{
var level = int.Parse(pieces[0]);
var position = int.Parse(pieces[1]);
var file = pieces[2];
var path = Path.GetDirectoryName(filename);
var ptablePath = Path.Combine(path, file);
ptable = PTable.FromFile(ptablePath);
ptable.VerifyFileHash();
CreateIfNeeded(level, tables);
tables[level].Insert(position, ptable);
}
catch (Exception exc)
{
// if PTable file path was correct, but data is corrupted, we still need to dispose opened streams
if (ptable != null)
{
ptable.Dispose();
}
// also dispose all previously loaded correct PTables
for (int i = 0; i < tables.Count; ++i)
{
for (int j = 0; j < tables[i].Count; ++j)
{
tables[i][j].Dispose();
}
}
throw new CorruptIndexException("Error while loading IndexMap.", exc);
}
}
}
return(new IndexMap(version, tables, prepareCheckpoint, commitCheckpoint, isHashCollision, maxTablesPerLevel));
}
