public static void DownloadCSV()
{
string time = @"" + DateTime.Now;
Directory.CreateDirectory(Directory.GetCurrentDirectory() + @"\Data");
do
{
Directory.CreateDirectory(Directory.GetCurrentDirectory() + @"\Data\_CurrentChunk");
WebClient wc = new WebClient();
string file = Directory.GetCurrentDirectory() + @"\Data\_CurrentChunk\" + @"BTCAnalysis_" + DateTime.Now.ToString("yyyy.MM.dd_HH-mm-ss-fff") + ".csv";
StreamWriter sw = new StreamWriter(file);
ChunkLogger.LOGIT("ChunkCounter: " + ChunkCounter);
ChunkLogger.LOGIT("Downloading Data | File: " + file);
sw.WriteLine(wc.DownloadString(url));
sw.Close();
ChunkCounter += 1;
if (ChunkCounter == 6)
{
ChunkCounter = 0;
Thread thread = new Thread(new ThreadStart(CreateChunk));
ChunkLogger.LOGIT("Chunk Creation | Staritng thread");
thread.Start();
}
WaitLib.wait.waitSec(1);
} while (true);
}
/// <summary>
/// Encrypts a file from its path and a plain password.
/// </summary>
/// <param name="inputFile"></param>
/// <param name="password"></param>
public static void FileEncrypt(string inputFile, string outputFile, string password)
{
//http://stackoverflow.com/questions/27645527/aes-encryption-on-large-files
ChunkLogger.LOGIT("---FileEncrypt()--- : Inputfile: " + inputFile + " | Outputfile: " + outputFile + " | Password: "******"What it does is repeatedly hash the user password along with the salt." High iteration counts.
var key = new Rfc2898DeriveBytes(passwordBytes, salt, 50000);
AES.Key = key.GetBytes(AES.KeySize / 8);
AES.IV = key.GetBytes(AES.BlockSize / 8);
//Cipher modes: http://security.stackexchange.com/questions/52665/which-is-the-best-cipher-mode-and-padding-mode-for-aes-encryption
AES.Mode = CipherMode.CFB;
// write salt to the begining of the output file, so in this case can be random every time
fsCrypt.Write(salt, 0, salt.Length);
CryptoStream cs = new CryptoStream(fsCrypt, AES.CreateEncryptor(), CryptoStreamMode.Write);
FileStream fsIn = new FileStream(inputFile, FileMode.Open);
//create a buffer (1mb) so only this amount will allocate in the memory and not the whole file
byte[] buffer = new byte[1048576];
int read;
try {
while ((read = fsIn.Read(buffer, 0, buffer.Length)) > 0)
{
//Application.DoEvents(); // -> for responsive GUI, using Task will be better!
cs.Write(buffer, 0, read);
}
// Close up
fsIn.Close();
} catch (Exception ex) {
Console.WriteLine("Error: " + ex.Message);
} finally {
cs.Close();
fsCrypt.Close();
}
}
/// <summary>
/// Decrypts an encrypted file with the FileEncrypt method through its path and the plain password.
/// </summary>
/// <param name="inputFile"></param>
/// <param name="outputFile"></param>
/// <param name="password"></param>
public static void FileDecrypt(string inputFile, string outputFile, string password)
{
ChunkLogger.LOGIT("---FileEncrypt()--- : Inputfile: " + inputFile + " | Outputfile: " + outputFile + " | Password: "******"CryptographicException error: " + ex_CryptographicException.Message);
} catch (Exception ex) {
Console.WriteLine("Error: " + ex.Message);
}
try {
cs.Close();
} catch (Exception ex) {
Console.WriteLine("Error by closing CryptoStream: " + ex.Message);
} finally {
fsOut.Close();
fsCrypt.Close();
}
}
public static void ExtractZipFile(string archiveFilenameIn, string password, string outFolder)
{
ChunkLogger.LOGIT("---ExtractZipFile()--- : outFolder: " + archiveFilenameIn + " | outFolder: " + outFolder + " | Password: "******"using" will close the stream even if an exception occurs.
using (FileStream streamWriter = File.Create(fullZipToPath)) {
StreamUtils.Copy(zipStream, streamWriter, buffer);
}
}
} finally {
if (zf != null)
{
zf.IsStreamOwner = true; // Makes close also shut the underlying stream
zf.Close(); // Ensure we release resources
}
}
}
// Recurses down the folder structure
//
private static void CompressFolder(string path, ZipOutputStream zipStream, int folderOffset)
{
ChunkLogger.LOGIT("---CompressFolder()--- : Path: " + path + " | zipStream: " + zipStream + " | folderoffset: " + folderOffset);
string[] files = Directory.GetFiles(path);
foreach (string filename in files)
{
FileInfo fi = new FileInfo(filename);
string entryName = filename.Substring(folderOffset); // Makes the name in zip based on the folder
entryName = ZipEntry.CleanName(entryName); // Removes drive from name and fixes slash direction
ZipEntry newEntry = new ZipEntry(entryName);
newEntry.DateTime = fi.LastWriteTime; // Note the zip format stores 2 second granularity
// Specifying the AESKeySize triggers AES encryption. Allowable values are 0 (off), 128 or 256.
// A password on the ZipOutputStream is required if using AES.
// newEntry.AESKeySize = 256;
// To permit the zip to be unpacked by built-in extractor in WinXP and Server2003, WinZip 8, Java, and other older code,
// you need to do one of the following: Specify UseZip64.Off, or set the Size.
// If the file may be bigger than 4GB, or you do not need WinXP built-in compatibility, you do not need either,
// but the zip will be in Zip64 format which not all utilities can understand.
// zipStream.UseZip64 = UseZip64.Off;
newEntry.Size = fi.Length;
zipStream.PutNextEntry(newEntry);
// Zip the file in buffered chunks
// the "using" will close the stream even if an exception occurs
byte[] buffer = new byte[4096];
using (FileStream streamReader = File.OpenRead(filename)) {
StreamUtils.Copy(streamReader, zipStream, buffer);
}
zipStream.CloseEntry();
zipStream.Close();
}
string[] folders = Directory.GetDirectories(path);
foreach (string folder in folders)
{
CompressFolder(folder, zipStream, folderOffset);
}
}
public static void CreateSample(string outPathname, string password, string folderName, int lvl)
{
ChunkLogger.LOGIT("---CreateSample()--- : outPathname: " + outPathname + " | folderName: " + folderName + " | password: "******" | LVL: " + lvl);
FileStream fsOut = File.Create(outPathname);
ZipOutputStream zipStream = new ZipOutputStream(fsOut);
zipStream.SetLevel(lvl); //0-9, 9 being the highest level of compression
zipStream.Password = password; // optional. Null is the same as not setting. Required if using AES.
// This setting will strip the leading part of the folder path in the entries, to
// make the entries relative to the starting folder.
// To include the full path for each entry up to the drive root, assign folderOffset = 0.
int folderOffset = folderName.Length + (folderName.EndsWith("\\") ? 0 : 1);
CompressFolder(folderName, zipStream, folderOffset);
zipStream.IsStreamOwner = true; // Makes the Close also Close the underlying stream
zipStream.Close();
}
public static void CreateChunk()
{
ChunkLogger.LOGIT("Creating Chunk");
string genFolderName = Directory.GetCurrentDirectory() + @"\Data\Chunk" + DateTime.Now.ToString("yyyy.MM.dd_HH-mm-ss");
Directory.CreateDirectory(genFolderName);
string[] allDatasetFullPath = Directory.GetFiles(Directory.GetCurrentDirectory() + @"\Data\_CurrentChunk");
string[] allDatasetFilenames = new string[allDatasetFullPath.Length];
ChunkLogger.LOGIT("All Datasets:");
for (int i = 0; i < allDatasetFullPath.Length; i++)
{
allDatasetFilenames[i] = Path.GetFileName(allDatasetFullPath[i]);
ChunkLogger.LOGIT(allDatasetFilenames[i]);
}
for (int i = 0; i < allDatasetFullPath.Length; i++)
{
File.Move(allDatasetFullPath[i], genFolderName + @"\" + allDatasetFilenames[i]);
}
string chunkFolderName = genFolderName;
ChunkGenerator.ZipToChunk.CreateSample(chunkFolderName + @".CompressedChunk.zip", ZipPassword, genFolderName, 6);
File.Move(chunkFolderName + @".CompressedChunk.zip", chunkFolderName + @".CompressedChunk");
ChunkLogger.LOGIT("Encrypting Chunk " + chunkFolderName + @".CompressedChunk");
//Encrypt CmpChnk
EncryptChunk(chunkFolderName + @".CompressedChunk", chunkFolderName + @".CmpChnk");
try {
Directory.Delete(chunkFolderName, true);
} catch (Exception r) {
ChunkLogger.LOGIT(r.ToString());
}
try {
File.Delete(chunkFolderName + @".CompressedChunk");
} catch (Exception r) {
ChunkLogger.LOGIT(r.ToString());
}
}
public static void DycryptAllChunk()
{
GCHandle gch = GCHandle.Alloc(AESPassword, GCHandleType.Pinned);
string CmpChnkPath = Directory.GetCurrentDirectory() + @"\Data\";
allCmpChnkFullPath = Directory.GetFiles(CmpChnkPath);
allCmpChnkFilenames = allCmpChnkFullPath;
allCmpChnkRAWNames = allCmpChnkFullPath;
for (int i = 0; i < allCmpChnkFilenames.Length; i++)
{
ChunkLogger.LOGIT("allCmpChnkFilenames: " + allCmpChnkFilenames[i]);
ChunkLogger.LOGIT("allCmpChnkFullPath: " + allCmpChnkFullPath[i]);
}
for (int i = 0; i < allCmpChnkFilenames.Length; i++)
{
allCmpChnkFilenames[i] = Path.GetFileName(allCmpChnkFilenames[i]);
ChunkLogger.LOGIT("Filename: " + allCmpChnkFilenames[i]);
}
char[] seperator;
seperator = ".".ToCharArray();
//Encryption
for (int i = 0; i < allCmpChnkFilenames.Length; i++)
{
ChunkLogger.LOGIT("Format: " + allCmpChnkFilenames[i]);
string[] splitStorage = allCmpChnkFilenames[i].Split(seperator);
ChunkLogger.LOGIT("AFTER SPLIT Format: " + allCmpChnkFilenames[i]);
for (int x = 0; x < splitStorage.Length; x++)
{
ChunkLogger.LOGIT("Splited: " + splitStorage[x]);
}
allCmpChnkRAWNames[i] = splitStorage[0] + "." + splitStorage[1] + "." + splitStorage[2];
ChunkLogger.LOGIT("");
ChunkLogger.LOGIT("RAW Name: " + allCmpChnkRAWNames[i]);
ChunkLogger.LOGIT("Filename: " + allCmpChnkFilenames[i]);
ChunkLogger.LOGIT("1st Path: " + CmpChnkPath + allCmpChnkFilenames[i]);
ChunkLogger.LOGIT("2nd Path: " + CmpChnkPath);
ChunkLogger.LOGIT("Password: "******"");
DycryptionForChunks.FileDecrypt(CmpChnkPath + allCmpChnkRAWNames[i] + ".CmpChnk", CmpChnkPath + allCmpChnkRAWNames[i] + ".CommpressedChunk", AESPassword);
}
//unzipping
ChunkLogger.LOGIT("Extracting");
for (int i = 0; i < allCmpChnkFullPath.Length; i++)
{
ChunkLogger.LOGIT("Extracting: " + allCmpChnkFilenames[i] + ".CommpressedChunk" + " | Target Folder: " + allCmpChnkRAWNames[i]);
Directory.CreateDirectory(CmpChnkPath + allCmpChnkRAWNames[i]);
ChunkGenerator.ZipToChunk.ExtractZipFile(CmpChnkPath + allCmpChnkRAWNames[i] + ".CommpressedChunk", ZipPassword, CmpChnkPath + allCmpChnkRAWNames[i]);
}
wait.waitSec(2);
ChunkLogger.LOGIT("Deleting junk");
for (int i = 0; i < allCmpChnkRAWNames.Length; i++)
{
ChunkLogger.LOGIT("operation " + (i + 1) + " of " + (allCmpChnkRAWNames.Length + 1));
ChunkLogger.LOGIT("Deleting: " + CmpChnkPath + allCmpChnkRAWNames[i] + ".CmpChnk");
//File.Delete(CmpChnkPath + allCmpChnkRAWNames[i] + ".CmpChnk");
ChunkLogger.LOGIT("Deleting" + CmpChnkPath + allCmpChnkRAWNames[i] + ".CommpressedChunk");
File.Delete(CmpChnkPath + allCmpChnkRAWNames[i] + ".CommpressedChunk");
}
}
