private static void fileWatch_Created(object sender, FileSystemEventArgs e)
{
bool IsCreated = false;
Console.WriteLine(e.FullPath + "was created:" + e.ChangeType.ToString());
var zipFile = Path.GetFileName(e.FullPath);
var dirName = Path.GetDirectoryName(e.FullPath);
var destFile = Path.Combine(dirName, Path.GetFileNameWithoutExtension(zipFile) + ".txt");
while (!IsCreated)
{
try
{
var result = new GZipResult();
GzipCompressFile.DecompressOneFile(e.FullPath, destFile, result);
if (!result.Errors)
{
IsCreated = true;
Console.WriteLine("解压成功");
}
else
{
Console.WriteLine("解压失败");
System.Threading.Thread.Sleep(5000);
}
}
catch (Exception ex)
{
System.Threading.Thread.Sleep(50);
}
}
Console.WriteLine("开始上传文件");
PutBlockToBlob(destFile);
Console.WriteLine("上传文件成功");
}
/// <summary>
/// Compress
/// </summary>
/// <param name="files">Array of FileInfo objects to be included in the zip file</param>
/// <param name="lpBaseFolder">Base folder to use when creating relative paths for the files
/// stored in the zip file. For example, if lpBaseFolder is 'C:\zipTest\Files\', and there is a file
/// 'C:\zipTest\Files\folder1\sample.txt' in the 'files' array, the relative path for sample.txt
/// will be 'folder1/sample.txt'</param>
/// <param name="lpDestFolder">Folder to write the zip file into</param>
/// <param name="zipFileName">Name of the zip file to write</param>
/// <param name="deleteTempFile">Boolean, true deleted the intermediate temp file, false leaves the temp file in lpDestFolder (for debugging)</param>
public static GZipResult Compress(FileInfo[] files, string lpBaseFolder, string lpDestFolder, string zipFileName, bool deleteTempFile)
{
GZipResult result = new GZipResult();
try
{
if (!lpDestFolder.EndsWith("\\"))
{
lpDestFolder += "\\";
}
string lpTempFile = lpDestFolder + zipFileName + ".tmp";
string lpZipFile = lpDestFolder + zipFileName;
result.TempFile = lpTempFile;
result.ZipFile = lpZipFile;
if (files != null && files.Length > 0)
{
CreateTempFile(files, lpBaseFolder, lpTempFile, result);
if (result.FileCount > 0)
{
CreateZipFile(lpTempFile, lpZipFile, result);
}
// delete the temp file
if (deleteTempFile)
{
File.Delete(lpTempFile);
result.TempFileDeleted = true;
}
}
}
catch //(Exception ex4)
{
result.Errors = true;
}
return result;
}
private static void CreateZipFile(string lpSourceFile, string lpZipFile, GZipResult result)
{
byte[] buffer;
int count = 0;
FileStream fsOut = null;
FileStream fsIn = null;
GZipStream gzip = null;
// compress the file into the zip file
try
{
fsOut = new FileStream(lpZipFile, FileMode.Create, FileAccess.Write, FileShare.None);
gzip = new GZipStream(fsOut, CompressionMode.Compress, true);
fsIn = new FileStream(lpSourceFile, FileMode.Open, FileAccess.Read, FileShare.Read);
buffer = new byte[4096];
//count = fsIn.Read(buffer, 0, buffer.Length);
do
{
count = fsIn.Read(buffer, 0, buffer.Length);
gzip.Write(buffer, 0, count);
}
while (count > 0);
fsIn.Close();
fsIn = null;
// compress to the zip file
// gzip.Write(buffer, 0, buffer.Length);
result.ZipFileSize = fsOut.Length;
result.CompressionPercent = GetCompressionPercent(result.TempFileSize, result.ZipFileSize);
}
catch //(Exception ex1)
{
result.Errors = true;
}
finally
{
if (gzip != null)
{
gzip.Close();
gzip = null;
}
if (fsOut != null)
{
fsOut.Close();
fsOut = null;
}
if (fsIn != null)
{
fsIn.Close();
fsIn = null;
}
}
}
private static void CreateTempFile(FileInfo[] files, string lpBaseFolder, string lpTempFile, GZipResult result)
{
byte[] buffer;
int count = 0;
byte[] header;
string fileHeader = null;
string fileModDate = null;
string lpFolder = null;
int fileIndex = 0;
string lpSourceFile = null;
string vpSourceFile = null;
GZipFileInfo gzf = null;
FileStream fsOut = null;
FileStream fsIn = null;
if (files != null && files.Length > 0)
{
try
{
result.Files = new GZipFileInfo[files.Length];
// open the temp file for writing
fsOut = new FileStream(lpTempFile, FileMode.Create, FileAccess.Write, FileShare.None);
foreach (FileInfo fi in files)
{
lpFolder = fi.DirectoryName + "\\";
try
{
gzf = new GZipFileInfo();
gzf.Index = fileIndex;
// read the source file, get its virtual path within the source folder
lpSourceFile = fi.FullName;
gzf.LocalPath = lpSourceFile;
vpSourceFile = lpSourceFile.Replace(lpBaseFolder, string.Empty);
vpSourceFile = vpSourceFile.Replace("\\", "/");
gzf.RelativePath = vpSourceFile;
fsIn = new FileStream(lpSourceFile, FileMode.Open, FileAccess.Read, FileShare.Read);
buffer = new byte[fsIn.Length];
count = fsIn.Read(buffer, 0, buffer.Length);
fsIn.Close();
fsIn = null;
fileModDate = fi.LastWriteTimeUtc.ToString();
gzf.ModifiedDate = fi.LastWriteTimeUtc;
gzf.Length = buffer.Length;
fileHeader = fileIndex.ToString() + "," + vpSourceFile + "," + fileModDate + "," + buffer.Length.ToString() + "\n";
header = Encoding.Default.GetBytes(fileHeader);
fsOut.Write(header, 0, header.Length);
fsOut.Write(buffer, 0, buffer.Length);
fsOut.WriteByte(10); // linefeed
gzf.AddedToTempFile = true;
// update the result object
result.Files[fileIndex] = gzf;
// increment the fileIndex
fileIndex++;
}
catch //(Exception ex1)
{
result.Errors = true;
}
finally
{
if (fsIn != null)
{
fsIn.Close();
fsIn = null;
}
}
if (fsOut != null)
{
result.TempFileSize = fsOut.Length;
}
}
}
catch //(Exception ex2)
{
result.Errors = true;
}
finally
{
if (fsOut != null)
{
fsOut.Close();
fsOut = null;
}
}
}
result.FileCount = fileIndex;
}
public static FileStream UnzipToTempFile(string lpZipFile, string lpTempFile, GZipResult result)
{
FileStream fsIn = null;
GZipStream gzip = null;
FileStream fsOut = null;
FileStream fsTemp = null;
const int bufferSize = 4096;
byte[] buffer = new byte[bufferSize];
int count = 0;
try
{
fsIn = new FileStream(lpZipFile, FileMode.Open, FileAccess.Read,
FileShare.Read);
result.ZipFileSize = fsIn.Length;
fsOut = new FileStream(lpTempFile, FileMode.Create, FileAccess.Write,
FileShare.None);
gzip = new GZipStream(fsIn, CompressionMode.Decompress, true);
while (true)
{
count = gzip.Read(buffer, 0, bufferSize);
if (count != 0)
{
fsOut.Write(buffer, 0, count);
}
if (count != bufferSize)
{
break;
}
}
}
catch //(Exception ex1)
{
result.Errors = true;
}
finally
{
if (gzip != null)
{
gzip.Close();
gzip = null;
}
if (fsOut != null)
{
fsOut.Close();
fsOut = null;
}
if (fsIn != null)
{
fsIn.Close();
fsIn = null;
}
}
fsTemp = new FileStream(lpTempFile, FileMode.Open, FileAccess.Read,
FileShare.None);
if (fsTemp != null)
{
result.TempFileSize = fsTemp.Length;
}
return fsTemp;
}
public static GZipResult Decompress(string lpSrcFolder, string lpDestFolder, string zipFileName, bool deleteTempFile, bool writeFiles, string addExtension, Hashtable htFiles, int bufferSize)
{
GZipResult result = new GZipResult();
if (!lpSrcFolder.EndsWith("\\"))
{
lpSrcFolder += "\\";
}
if (!lpDestFolder.EndsWith("\\"))
{
lpDestFolder += "\\";
}
string lpTempFile = lpSrcFolder + zipFileName + ".tmp";
string lpZipFile = lpSrcFolder + zipFileName;
result.TempFile = lpTempFile;
result.ZipFile = lpZipFile;
string line = null;
string lpFilePath = null;
string lpFolder = null;
GZipFileInfo gzf = null;
FileStream fsTemp = null;
ArrayList gzfs = new ArrayList();
bool write = false;
if (string.IsNullOrEmpty(addExtension))
{
addExtension = string.Empty;
}
else if (!addExtension.StartsWith("."))
{
addExtension = "." + addExtension;
}
// extract the files from the temp file
try
{
fsTemp = UnzipToTempFile(lpZipFile, lpTempFile, result);
if (fsTemp != null)
{
while (fsTemp.Position != fsTemp.Length)
{
line = null;
while (string.IsNullOrEmpty(line) && fsTemp.Position != fsTemp.Length)
{
line = ReadLine(fsTemp);
}
if (!string.IsNullOrEmpty(line))
{
gzf = new GZipFileInfo();
if (gzf.ParseFileInfo(line) && gzf.Length > 0)
{
gzfs.Add(gzf);
lpFilePath = lpDestFolder + gzf.RelativePath;
lpFolder = GetFolder(lpFilePath);
gzf.LocalPath = lpFilePath;
write = false;
if (htFiles == null || htFiles.ContainsKey(gzf.RelativePath))
{
gzf.RestoreRequested = true;
write = writeFiles;
}
if (write)
{
// make sure the folder exists
if (!Directory.Exists(lpFolder))
{
Directory.CreateDirectory(lpFolder);
}
// read from fsTemp and write out the file
gzf.Restored = WriteFile(fsTemp, gzf.Length, lpFilePath + addExtension, bufferSize);
}
else
{
// need to advance fsTemp
fsTemp.Position += gzf.Length;
}
}
}
}
}
}
catch //(Exception ex3)
{
result.Errors = true;
}
finally
{
if (fsTemp != null)
{
fsTemp.Close();
fsTemp = null;
}
}
// delete the temp file
try
{
if (deleteTempFile)
{
File.Delete(lpTempFile);
result.TempFileDeleted = true;
}
}
catch //(Exception ex4)
{
result.Errors = true;
}
result.FileCount = gzfs.Count;
result.Files = new GZipFileInfo[gzfs.Count];
gzfs.CopyTo(result.Files);
return result;
}
private static void fileWatch_Created(object sender, FileSystemEventArgs e)
{
System.Threading.Thread th = new System.Threading.Thread(() =>
{
bool IsCreated = false;
Console.WriteLine(e.FullPath + "was created:" + e.ChangeType.ToString());
UBA.Common.LogHelperNet.Error(e.FullPath + "was created:" + e.ChangeType.ToString(), null);
var zipFile = Path.GetFileName(e.FullPath);
var dirName = Path.GetDirectoryName(e.FullPath);
var destFile = Path.Combine(dirName, Path.GetFileNameWithoutExtension(zipFile) + ".txt");
while (!IsCreated)
{
try
{
System.Threading.Thread.Sleep(10000);
var result = new GZipResult();
GzipCompressFile.DecompressOneFile(e.FullPath, destFile, result);
if (!result.Errors)
{
IsCreated = true;
Console.WriteLine("解压成功");
UBA.Common.LogHelperNet.Info("解压成功", null);
}
else
{
Console.WriteLine("解压失败");
continue;
}
}
catch (Exception ex)
{
Console.WriteLine("解压失败");
UBA.Common.LogHelperNet.Info("解压失败" + ex.Message, null);
continue;
}
}
if (BlobUpload(destFile))
{
//TODO
UBA.Common.LogHelperNet.Info("开始创建HDI集群，并处理job", null);
UBA.Hadoop.CreateHDICluster.HadoopHelper.HadHoopStart();
UBA.Common.LogHelperNet.Info("提交job完成", null);
}
//BlobUpload(destFile);
});
th.Start();
}