async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
callback.TempFilesCleanupList.Add(sourceFile.Location);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var sourceFileInfo = new FileInfo(sourceFile.Location);
using (var inFileStream = sourceFileInfo.OpenRead())
using (var outFileStream = new FileStream(tmpFileName, FileMode.CreateNew))
using (var progress = sourceFileInfo.Length != 0 ? progressAggregator.CreateProgressSink() : (Progress.IProgressEventsSink)null)
{
using (var gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(inFileStream))
{
IOUtils.CopyStreamWithProgress(gzipStream, outFileStream, bytes =>
{
callback.SetStepDescription(string.Format("{1} {0}: Gunzipping...",
IOUtils.FileSizeToString(bytes), sourceFile.FullPath));
if (progress != null)
{
progress.SetValue((double)inFileStream.Position / (double)sourceFileInfo.Length);
}
}, callback.Cancellation);
return
(new PreprocessingStepParams(tmpFileName, sourceFile.FullPath,
sourceFile.PreprocessingHistory.Add(new PreprocessingHistoryItem(name))));
}
}
}
public async Task <Result <T> > GetAsync <T>(string url, NetworkCredential credentials = null, bool cacheResult = false)
{
try
{
var client = new WebClient();
client.Credentials = credentials;
client.Headers[HttpRequestHeader.AcceptEncoding] = "gzip";
var query = cacheResult ? url : AppendAntiCacheToken(url);
var result = await client.OpenReadTaskAsync(new Uri(query));
if (client.ResponseHeaders[HttpRequestHeader.ContentEncoding] == "gzip")
{
var gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(result);
return(new Result <T>(ReadObject <T>(gzipStream)));
}
return(new Result <T>(ReadObject <T>(result)));
}
catch (Exception ex)
{
if (ex is WebException && ((WebException)ex).Response != null && ((HttpWebResponse)((WebException)ex).Response).StatusCode == HttpStatusCode.Unauthorized)
{
return(new Result <T>(new UnauthorizedAccessException()));
}
return(new Result <T>(ex));
}
}
public IEnumerable <string> LoadEntries()
{
byte[] buff = new byte[256];
using (var compressedStream = bucketStreamProvider.OpenRead(address))
{
if (compressedStream == null || compressedStream.Length == 0)
{
yield break;
}
using (var zip = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(compressedStream))
using (var breader = new BinaryReader(zip))
{
var count = breader.ReadInt32();
for (int j = 0; j < count; ++j)
{
int cnt = breader.ReadInt32();
if (buff.Length < cnt)
{
buff = new byte[cnt + 1024];
}
int readed = 0;
while (readed != cnt)
{
readed += zip.Read(buff, readed, cnt - readed);
}
string message;
message = Encoding.UTF8.GetString(buff, 0, cnt);
yield return(message);
}
}
}
}
private void ExplodeZip()
{
string downloadedZipFile = folder + "\\tmp\\cifar-10-binary.tar.gz";
//GZipStream gzStream = new GZipStream(new FileStream(downloadedZipFile, FileMode.Open), CompressionMode.Decompress);
byte[] dataBuffer = new byte[4096];
using (System.IO.Stream fs = new FileStream(downloadedZipFile, FileMode.Open, FileAccess.Read))
{
using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(fs))
{
// Change this to your needs
string fnOut = Path.Combine(folder + "\\tmp", Path.GetFileNameWithoutExtension(downloadedZipFile));
using (FileStream fsOut = File.Create(fnOut))
{
ICSharpCode.SharpZipLib.Core.StreamUtils.Copy(gzipStream, fsOut, dataBuffer);
}
}
}
downloadedZipFile = folder + "\\tmp\\cifar-10-binary.tar";
using (System.IO.Stream fs = new FileStream(downloadedZipFile, FileMode.Open, FileAccess.Read))
{
ICSharpCode.SharpZipLib.Tar.TarArchive.CreateInputTarArchive(fs).ExtractContents(folder + "\\tmp");
}
}
private void _Check_GZ(string pathFileTAR, IDTSComponentEvents componentEvents)
{
bool b = false;
if (_testarchive)
{
using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gzip = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(System.IO.File.OpenRead(pathFileTAR)))
{
try
{
componentEvents.FireInformation(1, "UnZip SSIS", "Start verify file GZ (" + _folderSource + _fileZip + ")", null, 0, ref b);
gzip.CopyTo(System.IO.MemoryStream.Null);
gzip.Flush();
componentEvents.FireInformation(1, "UnZip SSIS", "File ZIP verified GZ (" + _folderSource + _fileZip + ")", null, 0, ref b);
}
catch (Exception ex)
{
throw new Exception("Verify file: " + _fileZip + " failed. (" + ex.Message + ")");
}
gzip.Close();
}
}
}
//@Brief Descomprime y comprueba el archivo de actualización
//@Return Boolean True si la operación se ha realizado correctamente
private bool UncompressUpdateFile()
{
bool bOk = false;
string tempFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonProgramFiles), "JBC\\Manager", System.Convert.ToString(My.Settings.Default.TempUpdateFolder));
string filePath = Path.Combine(tempFolder, System.Convert.ToString(My.Settings.Default.AppCompressFileName));
if (File.Exists(filePath))
{
string newFile = Path.ChangeExtension(filePath, "tar.gz");
try
{
File.Move(filePath, newFile);
//Descomprimir .tar.gz
Stream inStream = File.OpenRead(newFile);
Stream gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(inStream);
ICSharpCode.SharpZipLib.Tar.TarArchive tarArchive = ICSharpCode.SharpZipLib.Tar.TarArchive.CreateInputTarArchive(gzipStream);
tarArchive.ExtractContents(tempFolder);
tarArchive.Close();
gzipStream.Close();
inStream.Close();
//Comprobar que existe el .exe
bOk = Directory.GetFiles(tempFolder, "*.exe").Length > 0;
}
catch (Exception)
{
}
}
return(bOk);
}
private static Manifest ParseManifestStream(Stream secStream, FileEntry fileEntry, string password)
{
secStream.Seek(0, SeekOrigin.Begin);
using (var zStream = GetDecryptStream(secStream, fileEntry, password))
{
using (var gz = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(zStream))
{
var read = new MemoryStream();
var bytes = new byte[256];
do
{
var n = gz.Read(bytes, 0, bytes.Length);
if (n <= 0)
{
break;
}
read.Write(bytes, 0, n);
} while (true);
var data = read.ToArray();
// Debug.LogFormat("read manifest data {0}", data.Length);
var json = Encoding.UTF8.GetString(data);
return(JsonUtility.FromJson <Manifest>(json));
}
}
}
private void DownloadArchive(string url, string unpackDest) {
var downloadStream = new System.Net.WebClient().OpenRead(url);
var gzStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(downloadStream);
using (var file = System.IO.File.Create(unpackDest)) {
gzStream.CopyTo(file);
}
}
public static byte[] Dekompresja(this System.IO.MemoryStream input)
{
if (input == null || input.Length == 0)
{
return(null);
}
var lBytes = new List <byte>();
using (var zip = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(input))
{
byte[] buffer = new byte[4096];
while (true)
{
int read = zip.Read(buffer, 0, buffer.Length);
for (int i = 0; i < read; i++)
{
lBytes.Add(buffer[i]);
}
if (read <= 0)
{
break;
}
}
}
return(lBytes.ToArray());
}
/// <summary>
/// Decompresses the given data stream from its source ZIP or GZIP format.
/// </summary>
/// <param name="dataBytes"></param>
/// <returns></returns>
private static byte[] Inflate(byte[] dataBytes)
{
byte[] outputBytes = null;
var zipInputStream =
new ICSharpCode.SharpZipLib.Zip.ZipInputStream(new MemoryStream(dataBytes));
if (zipInputStream.CanDecompressEntry)
{
MemoryStream zipoutStream = new MemoryStream();
#if XBOX
byte[] buf = new byte[4096];
int amt = -1;
while (true)
{
amt = zipInputStream.Read(buf, 0, buf.Length);
if (amt == -1)
{
break;
}
zipoutStream.Write(buf, 0, amt);
}
#else
zipInputStream.CopyTo(zipoutStream);
#endif
outputBytes = zipoutStream.ToArray();
}
else
{
try {
var gzipInputStream =
new ICSharpCode.SharpZipLib.GZip.GZipInputStream(new MemoryStream(dataBytes));
MemoryStream zipoutStream = new MemoryStream();
#if XBOX
byte[] buf = new byte[4096];
int amt = -1;
while (true)
{
amt = gzipInputStream.Read(buf, 0, buf.Length);
if (amt == -1)
{
break;
}
zipoutStream.Write(buf, 0, amt);
}
#else
gzipInputStream.CopyTo(zipoutStream);
#endif
outputBytes = zipoutStream.ToArray();
}
catch (Exception exc)
{
CCLog.Log("Error decompressing image data: " + exc.Message);
}
}
return(outputBytes);
}
private void DownloadArchive(string url, string unpackDest)
{
var downloadStream = new System.Net.WebClient().OpenRead(url);
var gzStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(downloadStream);
using (var file = System.IO.File.Create(unpackDest)) {
gzStream.CopyTo(file);
}
}
public StreamReader DownloadResult(RawExtractionResult extractionResult)
{
DebugPrintLine("Download the report");
var streamResponse = extractionsContext.GetReadStream(extractionResult);
using (var gzip = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(streamResponse.Stream))
{
//Decompress data
return(new StreamReader(gzip, Encoding.UTF8));
}
}
private void _DeCompressGZ(string pathFileTAR, IDTSComponentEvents componentEvents)
{
_Check_GZ(pathFileTAR, componentEvents);
using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gzip = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(System.IO.File.OpenRead(pathFileTAR)))
{
_DeCompressTAR(pathFileTAR, componentEvents, gzip);
//gzip.Flush();
//gzip.Close();
}
}
private void iterateLines(JsonHandler handler)
{
var lineIndex = 0;
var fileLength = new FileInfo(_dumpFile).Length;
using (var fileStream = new FileStream(_dumpFile, FileMode.Open, FileAccess.Read))
{
using (var fileGzip = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(fileStream))
{
using (var file = new StreamReader(fileGzip))
{
var openingLine = file.ReadLine(); //skip first line with [
if (openingLine != "[")
{
throw new NotSupportedException();
}
var startTime = DateTime.Now;
while (!file.EndOfStream)
{
var currentPosition = fileStream.Position;
var line = file.ReadLine();
if (line == "]")
{
//last line
continue;
}
++lineIndex;
if (lineIndex % 1000 == 0)
{
var percentage = 100.0 * currentPosition / fileLength;
var currentDuration = DateTime.Now - startTime;
var expectedDuration = new TimeSpan((long)(currentDuration.Ticks / percentage * 100.0));
var remainingTime = expectedDuration - currentDuration;
Console.WriteLine("{0:0.00}% remaining time: {1:hh\\:mm\\:ss}", percentage, remainingTime);
}
if (line.EndsWith(','))
{
line = line.Substring(0, line.Length - 1);
}
var entity = JsonConvert.DeserializeObject <Dictionary <string, object> >(line);
handler(entity);
}
}
}
}
}
static public void WriteToFile(string path, string filename, int start, int byteslength)
{
List <string> AllOut = new List <string>();
if (File.Exists($"{path}/{filename}.txt"))
{
File.Delete($"{path}/{filename}.txt");
}
var newfile = File.CreateText($"{path}/{filename}.txt");
foreach (var filepath in Directory.GetFiles(path))
{
var file = Path.GetFileNameWithoutExtension(filepath);
var extension = Path.GetExtension(filepath);
if (extension == ".cha")
{
MemoryStream compressedStream = new MemoryStream(File.ReadAllBytes(filepath));
MemoryStream uncompressedStream = new MemoryStream();
ICSharpCode.SharpZipLib.GZip.GZipInputStream GZipOut = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(compressedStream);
GZipOut.CopyTo(uncompressedStream);
compressedStream.Close();
GZipOut.Close();
var buffer = uncompressedStream.ToArray();
//int SpriteCount = SpriteBuffer.Length / (SpriteRegionLength + SpriteSeparatorLength);
//var iterations = buffer.Length / (offsettonext + byteslength);
var iterations = 1;
for (int i = 0; i < iterations; i++)
{
Console.WriteLine($"Accessing {file}{i}...");
//offsettonext *= i;
string bytestring = "[";
byte[] ass = new byte[byteslength];
Array.Copy(buffer, ass, byteslength);
bytestring += BitConverter.ToString(ass);
bytestring += "]";
bytestring.Replace("-", " ");
var Output = $"header bytes for {file} is: \t" + bytestring + Environment.NewLine;
AllOut.Add(Output);
Console.WriteLine($"Wrote {file}{i}");
}
uncompressedStream.Close();
}
}
for (int i = 0; i < AllOut.Count; i++)
{
newfile.Write(AllOut[i]);
}
newfile.Close();
}
/// <summary>
/// 解压bytes
/// </summary>
/// <param name="param"></param>
/// <returns></returns>
public static byte[] DecompressBytes(byte[] buffer)
{
ICSharpCode.SharpZipLib.GZip.GZipInputStream gzp = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(new MemoryStream(buffer));
MemoryStream re = new MemoryStream();
int count = 0;
byte[] data = new byte[4096];
while ((count = gzp.Read(data, 0, data.Length)) != 0)
{
re.Write(data, 0, count);
}
return(re.ToArray());
}
private void DecodeContent()
{
if (ContentEncoding == ContentEncodings.None)
{
return;
}
System.IO.Stream s;
if (ContentEncoding == ContentEncodings.Deflate)
{
s = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(
new System.IO.MemoryStream(Body),
new ICSharpCode.SharpZipLib.Zip.Compression.Inflater(true));
}
else if (ContentEncoding == ContentEncodings.Gzip)
{
s = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(new System.IO.MemoryStream(Body));
}
else
{
throw new System.NotImplementedException("Unknown ContentEncoding of " + ContentEncoding);
}
System.IO.MemoryStream ms = new System.IO.MemoryStream();
int readChunkSize = 2048;
byte[] uncompressedChunk = new byte[readChunkSize];
int sizeRead;
do
{
sizeRead = s.Read(uncompressedChunk, 0, readChunkSize);
if (sizeRead > 0)
{
ms.Write(uncompressedChunk, 0, sizeRead);
}
else
{
break; // break out of the while loop
}
} while(sizeRead > 0);
s.Close();
// copy the CompressedBody to CompressedBody
CompressedBody = Body;
// extract the compressed body over the existing body
Body = ms.ToArray();
}
static void Convert(string file, Dictionary <ushort, ushort> mapping)
{
List <KeyValuePair <Vector3Int, byte[]> > chunks = new List <KeyValuePair <Vector3Int, byte[]> >();
using (FileStream baseStream = File.OpenRead(file))
using (var decompressor = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(baseStream))
using (BinaryReader binaryReader = new BinaryReader(decompressor)) {
while (binaryReader.ReadBoolean())
{
Vector3Int position = new Vector3Int(
binaryReader.ReadInt32(),
binaryReader.ReadInt32(),
binaryReader.ReadInt32()
);
int dataLength = binaryReader.ReadInt32();
byte[] data = binaryReader.ReadBytes(dataLength);
chunks.Add(new KeyValuePair <Vector3Int, byte[]>(position, data));
}
}
string fileName = Path.GetFileName(file);
Log.Write("Loaded {0}", fileName);
ushort[] raw = new ushort[4096];
using (FileStream baseStream = File.Open(Path.Combine(pathRegionFolderNew, fileName), FileMode.Create, FileAccess.Write))
using (var compressor = new ICSharpCode.SharpZipLib.GZip.GZipOutputStream(baseStream))
using (BinaryWriter binaryWriter = new BinaryWriter(compressor)) {
foreach (var pair in chunks)
{
FileSaver.FileCompression.DecompressRLEToRaw(pair.Value, raw);
for (int i = 0; i < 4096; i++)
{
raw[i] = mapping[raw[i]];
}
byte[] chunk = FileSaver.FileCompression.CompressRawToRLE(raw);
binaryWriter.Write(true);
binaryWriter.Write(pair.Key.x);
binaryWriter.Write(pair.Key.y);
binaryWriter.Write(pair.Key.z);
binaryWriter.Write(chunk.Length);
binaryWriter.Write(chunk);
}
binaryWriter.Write(false);
}
Log.Write("Converted {0}", fileName);
}
/// <summary>
/// Identifies the file using magic numbers.
/// </summary>
/// <returns>The filetype.</returns>
/// <param name="stream">Open stream to the file.</param>
public static FileType IdentifyFile(Stream stream)
{
FileType type = FileType.Unknown;
// Check the input.
if (stream == null)
{
return(type);
}
// Make sure the stream supports seeking.
if (!stream.CanSeek)
{
return(type);
}
// Start performing checks.
if (CheckGZip(stream))
{
// This may contain a tar file inside, create a new stream and check.
stream.Seek(0, SeekOrigin.Begin);
using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gz_stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(stream))
{
if (CheckTar(gz_stream))
{
type = FileType.TarGz;
}
else
{
type = FileType.GZip;
}
}
}
else if (CheckTar(stream))
{
type = FileType.Tar;
}
else if (CheckZip(stream))
{
type = FileType.Zip;
}
else if (CheckASCII(stream))
{
type = FileType.ASCII;
}
return(type);
}
static bool IsGzipFile(string filePath)
{
using (var fstm = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 64))
using (var stm = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(fstm))
{
try
{
stm.Read(new byte[0], 0, 0);
return(true);
}
catch (ICSharpCode.SharpZipLib.GZip.GZipException)
{
return(false);
}
}
}
private void DecompresNowGZip()
{
int restan = 0;
int size = 0;
byte[] BytesDecompressed = new byte[50000];
this.m_streaminput.Seek(0, SeekOrigin.Begin);
Stream s = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(this.m_streaminput);
try
{
while (true)
{
size = s.Read(BytesDecompressed, 0, (int)BytesDecompressed.Length);
if (size > 0)
{
restan += size;
size = restan;
}
else
{
break;
}
}
}
catch (ZipException e)
{
size = 0;
Debug.WriteLine(e.Message);
}
finally
{
s.Read(BytesDecompressed, 0, restan);
s.Flush();
this.m_streamoutput = null;
this.m_streamoutput = new MemoryStream(restan);
this.m_streamoutput.Write(BytesDecompressed, 0, restan);
this.m_byteoutput = null;
this.m_byteoutput = new byte[restan];
this.m_streamoutput.Seek(0, SeekOrigin.Begin);
this.m_streamoutput.Read(this.m_byteoutput, 0, restan);
this.m_streamoutput.Seek(0, SeekOrigin.Begin);
//s.Close();
}
}
/// <summary>
/// Decompress the update file
/// </summary>
/// <returns>True if the operation was successful</returns>
private bool UncompressUpdateFile()
{
bool bOk = false;
string sTempDir = Path.Combine(Path.GetTempPath(), System.Convert.ToString(My.Settings.Default.TempDirName));
string sTempFile = Path.Combine(sTempDir, System.Convert.ToString(My.Settings.Default.AppCompressFileName));
if (File.Exists(sTempFile))
{
string sTempCompressFile = Path.ChangeExtension(sTempFile, "tar.gz");
try
{
File.Move(sTempFile, sTempCompressFile);
//Descomprimir .tar.gz
Stream inStream = File.OpenRead(sTempCompressFile);
Stream gzipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(inStream);
ICSharpCode.SharpZipLib.Tar.TarArchive tarArchive = ICSharpCode.SharpZipLib.Tar.TarArchive.CreateInputTarArchive(gzipStream);
tarArchive.ExtractContents(sTempDir);
tarArchive.Close();
gzipStream.Close();
inStream.Close();
//Eliminamos el archivo comprimido
if (File.Exists(sTempCompressFile))
{
File.Delete(sTempCompressFile);
}
bOk = true;
}
catch (Exception ex)
{
LoggerModule.logger.Error(System.Reflection.MethodInfo.GetCurrentMethod().Name + ". Error: " + ex.Message);
}
}
else
{
LoggerModule.logger.Warn(System.Reflection.MethodInfo.GetCurrentMethod().Name + ". File doesn't exists: " + sTempFile);
}
return(bOk);
}
/// <summary>
/// 解压缩字节数组
/// 返回：已解压的字节数组
/// </summary>
/// <param name="data">待解压缩的字节数组</param>
/// <returns></returns>
public static byte[] DecompressBytes(byte[] data)
{
var o = new MemoryStream();
var s = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(new MemoryStream(data));
try
{
var size = 0;
var buf = new byte[1024];
while ((size = s.Read(buf, 0, buf.Length)) > 0)
{
o.Write(buf, 0, size);
}
}
finally
{
o.Close();
}
return(o.ToArray());
}
/// <summary>
/// 解压字符串
/// </summary>
/// <param name="param"></param>
/// <returns></returns>
public static string Decompress(string param)
{
string commonString = "";
byte[] buffer = Convert.FromBase64String(param);
MemoryStream ms = new MemoryStream(buffer);
Stream sm = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(ms);
//这里要指明要读入的格式，要不就有乱码
StreamReader reader = new StreamReader(sm, System.Text.Encoding.UTF8);
try
{
commonString = reader.ReadToEnd();
}
finally
{
sm.Close();
ms.Close();
}
return(commonString);
}
/// <summary>
/// Decompresses the specified string into the original string
/// </summary>
/// <param name="inputString">The compressed string</param>
/// <returns>The decompressed version of the specified string</returns>
public static string Decompress(string inputString)
{
// Create the zip stream
System.IO.MemoryStream memstream = new System.IO.MemoryStream(Convert.FromBase64String(inputString));
ICSharpCode.SharpZipLib.GZip.GZipInputStream zipstream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memstream);
Byte[] readbuffer = new Byte[1024];
System.IO.MemoryStream writebuffer = new System.IO.MemoryStream();
int bytesread = 1;
while (bytesread > 0)
{
bytesread = zipstream.Read(readbuffer, 0, readbuffer.Length);
writebuffer.Write(readbuffer, 0, bytesread);
}
writebuffer.Close();
zipstream.Close();
memstream.Close();
return(System.Text.Encoding.Unicode.GetString(writebuffer.ToArray()));
}
internal static byte[] Decompress(byte[] input)
{
int bufferSize = 2048;
try
{
MemoryStream ms = new MemoryStream(input);
MemoryStream ms1 = new MemoryStream();
ICSharpCode.SharpZipLib.GZip.GZipInputStream zipFile = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(ms);
byte[] output = new byte[2048];
while (bufferSize > 0)
{
bufferSize = zipFile.Read(output, 0, bufferSize);
ms1.Write(output, 0, bufferSize);
}
ms1.Close();
return(ms1.ToArray());
}
catch
{
return(null);
}
}
public int SaveGameData(string gameData, bool isCompressedAndBase64Encoded, out string message)
{
try
{
Data.GameDataset gameDataset = new Data.GameDataset();
//gameDataset.EnforceConstraints = false;
//gameDataset.
if (isCompressedAndBase64Encoded == true)
{
byte[] binaryGameData = Convert.FromBase64String(gameData);
MemoryStream memoryStream = new MemoryStream(binaryGameData);
ICSharpCode.SharpZipLib.GZip.GZipInputStream zipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream);
StreamReader streamReader = new StreamReader(zipStream, System.Text.Encoding.Unicode);
//string gameDataXml = streamReader.ReadToEnd();
gameDataset.ReadXml(streamReader, System.Data.XmlReadMode.IgnoreSchema);
}
else
{
//gameDataset.ReadXml(new StringReader(gameData));
gameDataset.ReadXml(new StringReader(gameData), System.Data.XmlReadMode.IgnoreSchema);
}
if (String.IsNullOrEmpty(Settings.Default.TagLastGameDataXmlFileLogLocation) == false)
{
File.WriteAllText(Path.Combine(Settings.Default.TagLastGameDataXmlFileLogLocation, Guid.NewGuid().ToString() + ".xml"), gameDataset.GetXml());
}
string currentIPAddress;
if (OperationContext.Current != null)
{
//http://nayyeri.net/detect-client-ip-in-wcf-3-5
OperationContext context = OperationContext.Current;
MessageProperties messageProperties = context.IncomingMessageProperties;
RemoteEndpointMessageProperty endpointProperty = (RemoteEndpointMessageProperty)messageProperties[RemoteEndpointMessageProperty.Name];
currentIPAddress = endpointProperty.Address;
}
else
{
currentIPAddress = "127.0.0.1"; // Supports unit tests.
}
int gameID = 0;
using (DataAccess.CSSDataContext db = new DataAccess.CSSDataContext())
{
using (DataAccess.CSSStatsDataContext statsDB = new DataAccess.CSSStatsDataContext())
{
var gameServer = statsDB.GameServers.FirstOrDefault(p => p.GameServerIPs.Where(r => r.IPAddress == currentIPAddress).Count() > 0);
if (gameServer == null)
{
throw new Exception("You may not upload data from this address: " + currentIPAddress);
}
try
{
foreach (Data.GameDataset.GameRow gameRow in gameDataset.Game)
{
gameID = SaveGame(db, statsDB, gameServer, gameRow);
}
}
catch (Exception ex)
{
if (String.IsNullOrEmpty(Settings.Default.TagExceptionLogFileName) == false)
{
File.AppendAllText(Settings.Default.TagExceptionLogFileName, DateTime.Now.ToString() + ": " + ex.ToString() + "\n\n\n");
}
throw;
}
statsDB.SubmitChanges();
db.SubmitChanges();
}
}
// Update AllegSkill rank.
AllegSkill.Calculator.UpdateAllegSkillForGame(gameID);
// Update Prestige Rank.
using (DataAccess.CSSStatsDataContext statsDB = new DataAccess.CSSStatsDataContext())
{
var game = statsDB.Games.FirstOrDefault(p => p.GameIdentID == gameID);
if (game == null)
{
Error.Write(new Exception("Tag.SaveGameData(): Couldn't get game for GameID: " + gameID));
}
else
{
PrestigeRankCalculator psc = new PrestigeRankCalculator();
psc.Calculate(statsDB, game);
}
}
message = "Game saved.";
return(gameID);
}
catch (Exception ex)
{
message = ex.ToString();
return(-1);
}
}
/// <summary>
/// Decompresses the specified bytes, using the specified compression type and output encoding
/// </summary>
/// <param name="bytes">The bytes to decompress.</param>
/// <param name="offset">The amount of offset to apply to the byte array before beginning decompression.</param>
/// <param name="length">The length of bytes to decompress in the byte array.</param>
/// <param name="compressionType">Type of the compression applied to the input string.</param>
/// <param name="outputEncoding">The output encoding to apply.</param>
/// <returns>Returns a string representing the uncompressed verison of the input</returns>
public static string Decompress(byte[] bytes, int offset, int length, CompressionType compressionType, Encoding outputEncoding)
{
if (bytes == null || bytes.Length == 0)
return string.Empty;
if (offset < 0)
throw new ArgumentOutOfRangeException("offset", "offset cannot be less than zero");
if (length > bytes.Length)
throw new ArgumentOutOfRangeException("length", "length cannot be greater than bytes.length");
if (length + offset > bytes.Length)
throw new ArgumentOutOfRangeException("length", "length + offset cannot be greater than bytes.length");
using (MemoryStream memoryStream = new MemoryStream(bytes)) {
Stream stream = null;
switch (compressionType) {
case CompressionType.BZip:
stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(memoryStream);
break;
case CompressionType.GZip:
stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream);
break;
// case CompressionType.Tar:
// stream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(memoryStream);
// break;
case CompressionType.Zip:
stream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(memoryStream);
break;
}
if (stream != null) {
var decoder = outputEncoding.GetDecoder();
StringBuilder builder = new StringBuilder();
byte[] buffer = new byte[2048];
char[] chars = new char[2048];
while (true) {
int size = stream.Read(buffer, 0, buffer.Length);
if (size == 0)
break;
int totalChars = decoder.GetChars(buffer, 0, size, chars, 0);
builder.Append(chars, 0, totalChars);
}
stream.Dispose();
return builder.ToString();
}
}
return string.Empty;
}
public override void ProcessCommand()
{
logger.Info("Processing CommandRequest_GetAniDBTitles");
try
{
bool process =
ServerSettings.AniDB_Username.Equals("jonbaby", StringComparison.InvariantCultureIgnoreCase) ||
ServerSettings.AniDB_Username.Equals("jmediamanager", StringComparison.InvariantCultureIgnoreCase);
if (!process)
{
return;
}
string url = Constants.AniDBTitlesURL;
logger.Trace("Get AniDB Titles: {0}", url);
Stream s = Utils.DownloadWebBinary(url);
int bytes = 2048;
byte[] data = new byte[2048];
StringBuilder b = new StringBuilder();
UTF8Encoding enc = new UTF8Encoding();
ICSharpCode.SharpZipLib.GZip.GZipInputStream zis = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(s);
while ((bytes = zis.Read(data, 0, data.Length)) > 0)
{
b.Append(enc.GetString(data, 0, bytes));
}
zis.Close();
string[] lines = b.ToString().Split('\n');
Dictionary <int, AnimeIDTitle> titles = new Dictionary <int, AnimeIDTitle>();
foreach (string line in lines)
{
if (line.Trim().Length == 0 || line.Trim().Substring(0, 1) == "#")
{
continue;
}
string[] fields = line.Split('|');
int animeID = 0;
int.TryParse(fields[0], out animeID);
if (animeID == 0)
{
continue;
}
string titleType = fields[1].Trim().ToLower();
//string language = fields[2].Trim().ToLower();
string titleValue = fields[3].Trim();
AnimeIDTitle thisTitle = null;
if (titles.ContainsKey(animeID))
{
thisTitle = titles[animeID];
}
else
{
thisTitle = new AnimeIDTitle();
thisTitle.AnimeIDTitleId = 0;
thisTitle.MainTitle = titleValue;
thisTitle.AnimeID = animeID;
titles[animeID] = thisTitle;
}
if (!string.IsNullOrEmpty(thisTitle.Titles))
{
thisTitle.Titles += "|";
}
if (titleType.Equals("1"))
{
thisTitle.MainTitle = titleValue;
}
thisTitle.Titles += titleValue;
}
foreach (AnimeIDTitle aniTitle in titles.Values)
{
//AzureWebAPI.Send_AnimeTitle(aniTitle);
CommandRequest_Azure_SendAnimeTitle cmdAzure =
new CommandRequest_Azure_SendAnimeTitle(aniTitle.AnimeID,
aniTitle.MainTitle, aniTitle.Titles);
cmdAzure.Save();
}
}
catch (Exception ex)
{
logger.Error("Error processing CommandRequest_GetAniDBTitles: {0}", ex.ToString());
return;
}
}
public int SaveGameData(string gameData, bool isCompressedAndBase64Encoded, out string message)
{
try
{
Data.GameDataset gameDataset = new Data.GameDataset();
//gameDataset.EnforceConstraints = false;
//gameDataset.
if (isCompressedAndBase64Encoded == true)
{
byte[] binaryGameData = Convert.FromBase64String(gameData);
MemoryStream memoryStream = new MemoryStream(binaryGameData);
ICSharpCode.SharpZipLib.GZip.GZipInputStream zipStream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream);
StreamReader streamReader = new StreamReader(zipStream, System.Text.Encoding.Unicode);
//string gameDataXml = streamReader.ReadToEnd();
gameDataset.ReadXml(streamReader, System.Data.XmlReadMode.IgnoreSchema);
}
else
{
//gameDataset.ReadXml(new StringReader(gameData));
gameDataset.ReadXml(new StringReader(gameData), System.Data.XmlReadMode.IgnoreSchema);
}
if (String.IsNullOrEmpty(Settings.Default.TagLastGameDataXmlFileLogLocation) == false)
File.WriteAllText(Path.Combine(Settings.Default.TagLastGameDataXmlFileLogLocation, Guid.NewGuid().ToString() + ".xml"), gameDataset.GetXml());
string currentIPAddress;
if (OperationContext.Current != null)
{
//http://nayyeri.net/detect-client-ip-in-wcf-3-5
OperationContext context = OperationContext.Current;
MessageProperties messageProperties = context.IncomingMessageProperties;
RemoteEndpointMessageProperty endpointProperty = (RemoteEndpointMessageProperty)messageProperties[RemoteEndpointMessageProperty.Name];
currentIPAddress = endpointProperty.Address;
}
else
currentIPAddress = "127.0.0.1"; // Supports unit tests.
int gameID = 0;
using (DataAccess.CSSDataContext db = new DataAccess.CSSDataContext())
{
using (DataAccess.CSSStatsDataContext statsDB = new DataAccess.CSSStatsDataContext())
{
var gameServer = statsDB.GameServers.FirstOrDefault(p => p.GameServerIPs.Where(r => r.IPAddress == currentIPAddress).Count() > 0);
if (gameServer == null)
throw new Exception("You may not upload data from this address: " + currentIPAddress);
try
{
foreach (Data.GameDataset.GameRow gameRow in gameDataset.Game)
gameID = SaveGame(db, statsDB, gameServer, gameRow);
}
catch (Exception ex)
{
if (String.IsNullOrEmpty(Settings.Default.TagExceptionLogFileName) == false)
File.AppendAllText(Settings.Default.TagExceptionLogFileName, DateTime.Now.ToString() + ": " + ex.ToString() + "\n\n\n");
throw;
}
statsDB.SubmitChanges();
db.SubmitChanges();
}
}
// Update AllegSkill rank.
AllegSkill.Calculator.UpdateAllegSkillForGame(gameID);
// Update Prestige Rank.
using (DataAccess.CSSStatsDataContext statsDB = new DataAccess.CSSStatsDataContext())
{
var game = statsDB.Games.FirstOrDefault(p => p.GameIdentID == gameID);
if (game == null)
{
Error.Write(new Exception("Tag.SaveGameData(): Couldn't get game for GameID: " + gameID));
}
else
{
PrestigeRankCalculator psc = new PrestigeRankCalculator();
psc.Calculate(statsDB, game);
}
}
message = "Game saved.";
return gameID;
}
catch (Exception ex)
{
message = ex.ToString();
return -1;
}
}
/// <summary>
/// Decompresses the given data stream from its source ZIP or GZIP format.
/// </summary>
/// <param name="dataBytes"></param>
/// <returns></returns>
private static byte[] Inflate(byte[] dataBytes)
{
byte[] outputBytes = null;
var zipInputStream =
new ICSharpCode.SharpZipLib.Zip.ZipInputStream(new MemoryStream(dataBytes));
if (zipInputStream.CanDecompressEntry) {
MemoryStream zipoutStream = new MemoryStream();
#if XBOX
byte[] buf = new byte[4096];
int amt = -1;
while (true)
{
amt = zipInputStream.Read(buf, 0, buf.Length);
if (amt == -1)
{
break;
}
zipoutStream.Write(buf, 0, amt);
}
#else
zipInputStream.CopyTo(zipoutStream);
#endif
outputBytes = zipoutStream.ToArray();
}
else {
try {
var gzipInputStream =
new ICSharpCode.SharpZipLib.GZip.GZipInputStream(new MemoryStream(dataBytes));
MemoryStream zipoutStream = new MemoryStream();
#if XBOX
byte[] buf = new byte[4096];
int amt = -1;
while (true)
{
amt = gzipInputStream.Read(buf, 0, buf.Length);
if (amt == -1)
{
break;
}
zipoutStream.Write(buf, 0, amt);
}
#else
gzipInputStream.CopyTo(zipoutStream);
#endif
outputBytes = zipoutStream.ToArray();
}
catch (Exception exc)
{
CCLog.Log("Error decompressing image data: " + exc.Message);
}
}
return outputBytes;
}
/// <summary>
/// Decompresses the specified bytes, using the specified compression type and output encoding
/// </summary>
/// <param name="bytes">The bytes to decompress.</param>
/// <param name="offset">The amount of offset to apply to the byte array before beginning decompression.</param>
/// <param name="length">The length of bytes to decompress in the byte array.</param>
/// <param name="compressionType">Type of the compression applied to the input string.</param>
/// <param name="outputEncoding">The output encoding to apply.</param>
/// <returns>Returns a string representing the uncompressed verison of the input</returns>
public static string Decompress(byte[] bytes, int offset, int length, CompressionType compressionType, Encoding outputEncoding)
{
if (bytes == null || bytes.Length == 0)
{
return(string.Empty);
}
if (offset < 0)
{
throw new ArgumentOutOfRangeException("offset", "offset cannot be less than zero");
}
if (length > bytes.Length)
{
throw new ArgumentOutOfRangeException("length", "length cannot be greater than bytes.length");
}
if (length + offset > bytes.Length)
{
throw new ArgumentOutOfRangeException("length", "length + offset cannot be greater than bytes.length");
}
using (MemoryStream memoryStream = new MemoryStream(bytes)) {
Stream stream = null;
switch (compressionType)
{
case CompressionType.BZip:
stream = new ICSharpCode.SharpZipLib.BZip2.BZip2InputStream(memoryStream);
break;
case CompressionType.GZip:
stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(memoryStream);
break;
// case CompressionType.Tar:
// stream = new ICSharpCode.SharpZipLib.Tar.TarInputStream(memoryStream);
// break;
case CompressionType.Zip:
stream = new ICSharpCode.SharpZipLib.Zip.Compression.Streams.InflaterInputStream(memoryStream);
break;
}
if (stream != null)
{
var decoder = outputEncoding.GetDecoder();
StringBuilder builder = new StringBuilder();
byte[] buffer = new byte[2048];
char[] chars = new char[2048];
while (true)
{
int size = stream.Read(buffer, 0, buffer.Length);
if (size == 0)
{
break;
}
int totalChars = decoder.GetChars(buffer, 0, size, chars, 0);
builder.Append(chars, 0, totalChars);
}
stream.Dispose();
return(builder.ToString());
}
}
return(string.Empty);
}
private static FileStream UnzipToTempFile(string lpZipFile, string lpTempFile, ref GZipResult result)
{
FileStream fsIn = null;
ICSharpCode.SharpZipLib.GZip.GZipInputStream gzip = null;
FileStream fsOut = null;
FileStream fsTemp = null;
const int bufferSize = 4096;
byte[] buffer = new byte[bufferSize];
int count = 0;
try
{
fsIn = new FileStream(lpZipFile, FileMode.Open, FileAccess.Read, FileShare.Read);
result.ZipFileSize = fsIn.Length;
fsOut = new FileStream(lpTempFile, FileMode.Create, FileAccess.Write, FileShare.None);
gzip = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(fsIn);
// gzip = new GZipStream(fsIn, CompressionMode.Decompress, true);
while (true)
{
count = gzip.Read(buffer, 0, bufferSize);
if (count != 0)
{
fsOut.Write(buffer, 0, count);
}
if (count != bufferSize)
{
break;
}
}
}
catch (System.Exception ex1)
{ //(Exception ex1)
Debug.Log("UnzipToTempFile is Fail!!! " + ex1.ToString());
result.Errors = true;
}
finally
{
if (gzip != null)
{
gzip.Close();
gzip = null;
}
if (fsOut != null)
{
fsOut.Close();
fsOut = null;
}
if (fsIn != null)
{
fsIn.Close();
fsIn = null;
}
}
fsTemp = new FileStream(lpTempFile, FileMode.Open, FileAccess.Read, FileShare.None);
if (fsTemp != null)
{
result.TempFileSize = fsTemp.Length;
}
return(fsTemp);
}
public override void ProcessCommand()
{
logger.Info("Processing CommandRequest_GetAniDBTitles");
try
{
bool process = (ServerSettings.AniDB_Username.Equals("jonbaby", StringComparison.InvariantCultureIgnoreCase) ||
ServerSettings.AniDB_Username.Equals("jmediamanager", StringComparison.InvariantCultureIgnoreCase));
if (!process) return;
string url = Constants.AniDBTitlesURL;
logger.Trace("Get AniDB Titles: {0}", url);
Stream s = Utils.DownloadWebBinary(url);
int bytes = 2048;
byte[] data = new byte[2048];
StringBuilder b = new StringBuilder();
UTF8Encoding enc = new UTF8Encoding();
ICSharpCode.SharpZipLib.GZip.GZipInputStream zis = new ICSharpCode.SharpZipLib.GZip.GZipInputStream(s);
while ((bytes = zis.Read(data, 0, data.Length)) > 0)
b.Append(enc.GetString(data, 0, bytes));
zis.Close();
AniDB_Anime_TitleRepository repTitles = new AniDB_Anime_TitleRepository();
string[] lines = b.ToString().Split('\n');
Dictionary<int, AnimeIDTitle> titles = new Dictionary<int, AnimeIDTitle>();
foreach (string line in lines)
{
if (line.Trim().Length == 0 || line.Trim().Substring(0, 1) == "#") continue;
string[] fields = line.Split('|');
int animeID = 0;
int.TryParse(fields[0], out animeID);
if (animeID == 0) continue;
string titleType = fields[1].Trim().ToLower();
//string language = fields[2].Trim().ToLower();
string titleValue = fields[3].Trim();
AnimeIDTitle thisTitle = null;
if (titles.ContainsKey(animeID))
{
thisTitle = titles[animeID];
}
else
{
thisTitle = new AnimeIDTitle();
thisTitle.AnimeIDTitleId = 0;
thisTitle.MainTitle = titleValue;
thisTitle.AnimeID = animeID;
titles[animeID] = thisTitle;
}
if (!string.IsNullOrEmpty(thisTitle.Titles))
thisTitle.Titles += "|";
if (titleType.Equals("1"))
thisTitle.MainTitle = titleValue;
thisTitle.Titles += titleValue;
}
foreach (AnimeIDTitle aniTitle in titles.Values)
{
//AzureWebAPI.Send_AnimeTitle(aniTitle);
CommandRequest_Azure_SendAnimeTitle cmdAzure = new CommandRequest_Azure_SendAnimeTitle(aniTitle.AnimeID, aniTitle.MainTitle, aniTitle.Titles);
cmdAzure.Save();
}
}
catch (Exception ex)
{
logger.Error("Error processing CommandRequest_GetAniDBTitles: {0}", ex.ToString());
return;
}
}
/// <summary>
/// Identifies the file using magic numbers.
/// </summary>
/// <returns>The filetype.</returns>
/// <param name="stream">Open stream to the file.</param>
public static FileType IdentifyFile(Stream stream)
{
FileType type = FileType.Unknown;
// Check the input.
if (stream == null)
{
return type;
}
// Make sure the stream supports seeking.
if (!stream.CanSeek)
{
return type;
}
// Start performing checks.
if (CheckGZip(stream))
{
// This may contain a tar file inside, create a new stream and check.
stream.Seek (0, SeekOrigin.Begin);
using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gz_stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream (stream))
{
if (CheckTar(gz_stream))
{
type = FileType.TarGz;
}
else
{
type = FileType.GZip;
}
}
}
else if (CheckTar(stream))
{
type = FileType.Tar;
}
else if (CheckZip(stream))
{
type = FileType.Zip;
}
else if (CheckASCII(stream))
{
type = FileType.ASCII;
}
return type;
}
/// <summary>
/// Decompresses the given data stream from its source ZIP or GZIP format.
/// </summary>
/// <param name="dataBytes"></param>
/// <returns></returns>
private static byte[] Inflate(byte[] dataBytes)
{
byte[] outputBytes = null;
var zipInputStream =
new ICSharpCode.SharpZipLib.Zip.ZipInputStream(new MemoryStream(dataBytes));
if (zipInputStream.CanDecompressEntry) {
MemoryStream zipoutStream = new MemoryStream();
zipInputStream.CopyTo(zipoutStream);
outputBytes = zipoutStream.ToArray();
}
else {
try {
var gzipInputStream =
new ICSharpCode.SharpZipLib.GZip.GZipInputStream(new MemoryStream(dataBytes));
MemoryStream zipoutStream = new MemoryStream();
gzipInputStream.CopyTo(zipoutStream);
outputBytes = zipoutStream.ToArray();
}
catch (Exception exc)
{
CCLog.Log("Error decompressing image data: " + exc.Message);
}
}
return outputBytes;
}