private static byte[] BZip2Decompress(Stream data, int expectedLength)
{
using (MemoryStream output = new MemoryStream(expectedLength))
{
#if WITH_DOTNETZIP
using (var stream = new Ionic.BZip2.BZip2InputStream(data, false))
{
stream.CopyTo(output);
}
#elif WITH_BZIP2NET
using (var stream = new Bzip2.BZip2InputStream(data, false))
{
stream.CopyTo(output);
}
#elif WITH_SHARPCOMPRESS
/*
* using (var stream = new SharpCompress.Compressors.BZip2.BZip2Stream(data, SharpCompress.Compressors.CompressionMode.Decompress, false))
* {
* stream.CopyTo(output);
* }
*/
#elif WITH_SHARPZIPLIB
ICSharpCode.SharpZipLib.BZip2.BZip2.Decompress(data, output, true);
#else
throw new NotImplementedException("Please define which compression library you want to use");
#endif
return(output.ToArray());
}
}
public void CompressFiles()
{
var fileSelector = new FileSelector("");
var readOptions = new ReadOptions();
var crc32 = new CRC32();
var inputStream = new Ionic.BZip2.BZip2InputStream(new FileStream("", FileMode.Open));
}
public byte[] Transform(byte[] content)
{
using (var contentStream = new MemoryStream(content))
using (var bzipStream = new Ionic.BZip2.BZip2InputStream(contentStream))
using (var memoryStreamDecompressed = new MemoryStream())
{
bzipStream.CopyTo(memoryStreamDecompressed);
var bytes = memoryStreamDecompressed.ToArray();
return bytes;
}
}
private static byte[] BZip2Decompress(Stream data, int expectedLength)
{
using (MemoryStream output = new MemoryStream(expectedLength))
{
using (var stream = new Ionic.BZip2.BZip2InputStream(data))
{
stream.CopyTo(output);
}
return(output.ToArray());
}
}
/// <summary>
/// Decompresses the input stream.
/// </summary>
/// <param name="data">Stream containing compressed data.</param>
/// <param name="expectedLength">The expected length (in bytes) of the decompressed data.</param>
/// <returns>Byte array containing the decompressed data.</returns>
public static byte[] Decompress(Stream data, uint expectedLength)
{
using (var output = new MemoryStream((int)expectedLength))
{
#if true
using var bZip2InputStream = new Ionic.BZip2.BZip2InputStream(data, true);
bZip2InputStream.CopyTo(output);
#else
ICSharpCode.SharpZipLib.BZip2.BZip2.Decompress(data, output, false);
#endif
return(output.ToArray());
}
}
public JContainer CopyCommand(dynamic cmd)
{
using (log4net.NDC.Push(Guid.NewGuid().ToString()))
{
// Log command *after* we've removed security details from the command.
bool result = false;
string details = null;
object newData = null;
try
{
dynamic timeout = cmd.wait; // TODO: Useful?
TemplateObjectTO srcTemplateObjectTO = TemplateObjectTO.ParseJson(cmd.srcTO);
TemplateObjectTO destTemplateObjectTO = TemplateObjectTO.ParseJson(cmd.destTO);
VolumeObjectTO destVolumeObjectTO = VolumeObjectTO.ParseJson(cmd.destTO);
logger.Info(CloudStackTypes.CopyCommand + cmd.ToString());
string destFile = null;
if (destTemplateObjectTO != null && destTemplateObjectTO.primaryDataStore != null)
{
destFile = destTemplateObjectTO.FullFileName;
if (!destTemplateObjectTO.primaryDataStore.isLocal)
{
PrimaryDataStoreTO primary = destTemplateObjectTO.primaryDataStore;
Utils.ConnectToRemote(primary.UncPath, primary.Domain, primary.User, primary.Password);
}
}
// Template already downloaded?
if (destFile != null && File.Exists(destFile) &&
!String.IsNullOrEmpty(destTemplateObjectTO.checksum))
{
// TODO: checksum fails us, because it is of the compressed image.
// ASK: should we store the compressed or uncompressed version or is the checksum not calculated correctly?
logger.Debug(CloudStackTypes.CopyCommand + " calling VerifyChecksum to see if we already have the file at " + destFile);
result = VerifyChecksum(destFile, destTemplateObjectTO.checksum);
if (!result)
{
result = true;
logger.Debug(CloudStackTypes.CopyCommand + " existing file has different checksum " + destFile);
}
}
// Do we have to create a new one?
if (!result)
{
// Create local copy of a template?
if (srcTemplateObjectTO != null && destTemplateObjectTO != null)
{
// S3 download to primary storage?
// NFS provider download to primary storage?
if ((srcTemplateObjectTO.s3DataStoreTO != null || srcTemplateObjectTO.nfsDataStoreTO != null) && destTemplateObjectTO.primaryDataStore != null)
{
if (File.Exists(destFile))
{
logger.Info("Deleting existing file " + destFile);
File.Delete(destFile);
}
if (srcTemplateObjectTO.s3DataStoreTO != null)
{
// Download from S3 to destination data storage
DownloadS3ObjectToFile(srcTemplateObjectTO.path, srcTemplateObjectTO.s3DataStoreTO, destFile);
}
else if (srcTemplateObjectTO.nfsDataStoreTO != null)
{
// Download from S3 to destination data storage
Utils.DownloadCifsFileToLocalFile(srcTemplateObjectTO.path, srcTemplateObjectTO.nfsDataStoreTO, destFile);
}
// Uncompress, as required
if (srcTemplateObjectTO.path.EndsWith(".bz2"))
{
String uncompressedFile = destFile + ".tmp";
String compressedFile = destFile;
using (var uncompressedOutStrm = new FileStream(uncompressedFile, FileMode.CreateNew, FileAccess.Write))
{
using (var compressedInStrm = new FileStream(destFile, FileMode.Open, FileAccess.Read))
{
using (var bz2UncompressorStrm = new Ionic.BZip2.BZip2InputStream(compressedInStrm, true) /* outer 'using' statement will close FileStream*/ )
{
int count = 0;
int bufsize = 1024 * 1024;
byte[] buf = new byte[bufsize];
// EOF returns -1, see http://dotnetzip.codeplex.com/workitem/16069
while (0 < (count = bz2UncompressorStrm.Read(buf, 0, bufsize)))
{
uncompressedOutStrm.Write(buf, 0, count);
}
}
}
}
File.Delete(compressedFile);
File.Move(uncompressedFile, compressedFile);
if (File.Exists(uncompressedFile))
{
String errMsg = "Extra file left around called " + uncompressedFile + " when creating " + destFile;
logger.Error(errMsg);
throw new IOException(errMsg);
}
}
// assert
if (!File.Exists(destFile))
{
String errMsg = "Failed to create " + destFile + " , because the file is missing";
logger.Error(errMsg);
throw new IOException(errMsg);
}
newData = cmd.destTO;
result = true;
}
else
{
details = "Data store combination not supported";
}
}
// Create volume from a template?
else if (srcTemplateObjectTO != null && destVolumeObjectTO != null)
{
// VolumeObjectTO guesses file extension based on existing files
// this can be wrong if the previous file had a different file type
var guessedDestFile = destVolumeObjectTO.FullFileName;
if (File.Exists(guessedDestFile))
{
logger.Info("Deleting existing file " + guessedDestFile);
File.Delete(guessedDestFile);
}
destVolumeObjectTO.format = srcTemplateObjectTO.format;
destFile = destVolumeObjectTO.FullFileName;
if (File.Exists(destFile))
{
logger.Info("Deleting existing file " + destFile);
File.Delete(destFile);
}
string srcFile = srcTemplateObjectTO.FullFileName;
if (!File.Exists(srcFile))
{
details = "Local template file missing from " + srcFile;
}
else
{
// TODO: thin provision instead of copying the full file.
File.Copy(srcFile, destFile);
newData = cmd.destTO;
result = true;
}
}
else
{
details = "Data store combination not supported";
}
}
}
catch (Exception ex)
{
// Test by providing wrong key
details = CloudStackTypes.CopyCommand + " failed on exception, " + ex.Message;
logger.Error(details, ex);
}
object ansContent = new
{
result = result,
details = details,
newData = cmd.destTO,
contextMap = contextMap
};
return ReturnCloudStackTypedJArray(ansContent, CloudStackTypes.CopyCmdAnswer);
}
}
public async Task <bool> DoWork(IProgress <DownloadProgressReportModel> progressSent, CancellationToken cancellationToken)
{
DownloadProgressReportModel report;
token = cancellationToken;
progress = progressSent;
string fileName = "invTypes.xls.bz2";
if (File.Exists(fileName))
{
File.Delete(fileName);
}
WebClient wb = new WebClient();
wb.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)");
wb.DownloadFile("https://www.fuzzwork.co.uk/dump/latest/invTypes.xls.bz2", fileName);
var fileNameXls = Path.GetFileNameWithoutExtension(fileName);
if (File.Exists(fileNameXls))
{
File.Delete(fileNameXls);
}
using (Stream fs = File.OpenRead(fileName), output = File.Create(fileNameXls), decompressor = new Ionic.BZip2.BZip2InputStream(fs))
{
byte[] buffer = new byte[2048];
int n;
while ((n = decompressor.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, n);
}
}
File.Delete(fileName);
List <List <string> > allItemList = new List <List <string> >();
startTime = DateTimeOffset.Now.ToUnixTimeSeconds();
using (var stream = File.Open(fileNameXls, FileMode.Open, FileAccess.Read))
{
using (var reader = ExcelReaderFactory.CreateReader(stream))
{
var conf = new ExcelDataSetConfiguration
{
ConfigureDataTable = _ => new ExcelDataTableConfiguration
{
UseHeaderRow = true
}
};
var dataSet = reader.AsDataSet(conf);
var dataTable = dataSet.Tables[0];
int totalItems = dataTable.Rows.Count;
for (int i = 0; i < totalItems; i++)
{
List <string> singleRow = new List <string>();
singleRow.Add(dataTable.Rows[i][0].ToString());
singleRow.Add(dataTable.Rows[i][2].ToString());
string itemVolumeLocal = dataTable.Rows[i][5].ToString();
string itemVolumeInternational = itemVolumeLocal.Replace(',', '.');
singleRow.Add(itemVolumeInternational);
allItemList.Add(singleRow);
token.ThrowIfCancellationRequested();
int progressPercentage = Convert.ToInt32(((double)i / totalItems) * 100);
report = new DownloadProgressReportModel();
report.PercentageComplete = progressPercentage;
report.MessageRemaining = "Step 2/4: Processing new item list" + StaticMethods.EstimatedTime(startTime, i, totalItems);
progress.Report(report);
}
}
}
File.Delete(fileNameXls);
if (allItemList.Any())
{
List <ItemModel> itemsDB = SqliteDataAccess.LoadItems();
List <ItemModel> todoItems = new List <ItemModel>();
ItemFiltersModel filters = SqliteDataAccess.LoadItemFilters();
long currentTImestamp = DateTimeOffset.Now.ToUnixTimeSeconds();
startedItem = 0;
totalItems = allItemList.Count;
startTime = DateTimeOffset.Now.ToUnixTimeSeconds();
foreach (List <string> newItem in allItemList)
{
token.ThrowIfCancellationRequested();
bool isNewItem = true;
foreach (ItemModel item in itemsDB)
{
if (Int32.Parse(newItem[0]) == item.type_id)
{
isNewItem = false;
long oldTimestamp = item.updated_at + (filters.updated_item_max_age * 24 * 60 * 60);
if (currentTImestamp > oldTimestamp)
{
ItemModel newTodoItem = new ItemModel();
newTodoItem.type_id = item.type_id;
newTodoItem.name = newItem[1];
newTodoItem.volume = Decimal.Parse(newItem[2], CultureInfo.InvariantCulture);
todoItems.Add(newTodoItem);
}
}
}
if (isNewItem)
{
ItemModel newTodoItem = new ItemModel();
newTodoItem.type_id = Int32.Parse(newItem[0]);
newTodoItem.name = newItem[1];
newTodoItem.volume = Decimal.Parse(newItem[2], CultureInfo.InvariantCulture);
newTodoItem.sell_price = 0;
newTodoItem.trade_volume = 0;
newTodoItem.updated_at = 0;
SqliteDataAccess.SaveItem(newTodoItem);
todoItems.Add(newTodoItem);
}
report = new DownloadProgressReportModel();
report.PercentageComplete = Convert.ToInt32(((double)startedItem / totalItems) * 100);
report.MessageRemaining = "Step 3/4: Checking items in database" + StaticMethods.EstimatedTime(startTime, startedItem, totalItems);
progress.Report(report);
startedItem++;
}
startTime = DateTimeOffset.Now.ToUnixTimeSeconds();
totalItems = todoItems.Count();
startedItem = 0;
finishedItem = 0;
maxAsync = 0;
while (startedItem < totalItems)
{
token.ThrowIfCancellationRequested();
if (maxAsync < filters.max_async_tasks)
{
maxAsync++;
Task.Run(() => GetAndUpdateItem(todoItems[startedItem]));
}
Thread.Sleep(100);
}
}
return(finishedWithErrors);
}
public JContainer CopyCommand(dynamic cmd)
{
using (log4net.NDC.Push(Guid.NewGuid().ToString()))
{
// Log command *after* we've removed security details from the command.
logger.Info(CloudStackTypes.CopyCommand + Utils.CleanString(cmd.ToString()));
bool result = false;
string details = null;
object newData = null;
TemplateObjectTO destTemplateObjectTO = null;
VolumeObjectTO destVolumeObjectTO = null;
VolumeObjectTO srcVolumeObjectTO = null;
TemplateObjectTO srcTemplateObjectTO = null;
try
{
srcTemplateObjectTO = TemplateObjectTO.ParseJson(cmd.srcTO);
destTemplateObjectTO = TemplateObjectTO.ParseJson(cmd.destTO);
srcVolumeObjectTO = VolumeObjectTO.ParseJson(cmd.srcTO);
destVolumeObjectTO = VolumeObjectTO.ParseJson(cmd.destTO);
string destFile = null;
if (destTemplateObjectTO != null)
{
if (destTemplateObjectTO.path == null)
{
destTemplateObjectTO.path = System.Guid.NewGuid().ToString();
}
if (destTemplateObjectTO.primaryDataStore != null)
{
destFile = destTemplateObjectTO.FullFileName;
}
else if (destTemplateObjectTO.nfsDataStoreTO != null)
{
destFile = destTemplateObjectTO.FullFileName;
}
}
// Create local copy of a template?
if (srcTemplateObjectTO != null && destTemplateObjectTO != null)
{
// S3 download to primary storage?
// NFS provider download to primary storage?
if ((srcTemplateObjectTO.s3DataStoreTO != null || srcTemplateObjectTO.nfsDataStoreTO != null) && destTemplateObjectTO.primaryDataStore != null)
{
if (File.Exists(destFile))
{
logger.Info("Deleting existing file " + destFile);
File.Delete(destFile);
}
if (srcTemplateObjectTO.s3DataStoreTO != null)
{
// Download from S3 to destination data storage
DownloadS3ObjectToFile(srcTemplateObjectTO.path, srcTemplateObjectTO.s3DataStoreTO, destFile);
}
else if (srcTemplateObjectTO.nfsDataStoreTO != null)
{
// Download from S3 to destination data storage
Utils.DownloadCifsFileToLocalFile(srcTemplateObjectTO.path, srcTemplateObjectTO.nfsDataStoreTO, destFile);
}
// Uncompress, as required
if (srcTemplateObjectTO.path.EndsWith(".bz2"))
{
String uncompressedFile = destFile + ".tmp";
String compressedFile = destFile;
using (var uncompressedOutStrm = new FileStream(uncompressedFile, FileMode.CreateNew, FileAccess.Write))
{
using (var compressedInStrm = new FileStream(destFile, FileMode.Open, FileAccess.Read))
{
using (var bz2UncompressorStrm = new Ionic.BZip2.BZip2InputStream(compressedInStrm, true) /* outer 'using' statement will close FileStream*/ )
{
int count = 0;
int bufsize = 1024 * 1024;
byte[] buf = new byte[bufsize];
// EOF returns -1, see http://dotnetzip.codeplex.com/workitem/16069
while (0 < (count = bz2UncompressorStrm.Read(buf, 0, bufsize)))
{
uncompressedOutStrm.Write(buf, 0, count);
}
}
}
}
File.Delete(compressedFile);
File.Move(uncompressedFile, compressedFile);
if (File.Exists(uncompressedFile))
{
String errMsg = "Extra file left around called " + uncompressedFile + " when creating " + destFile;
logger.Error(errMsg);
throw new IOException(errMsg);
}
}
// assert
if (!File.Exists(destFile))
{
String errMsg = "Failed to create " + destFile + " , because the file is missing";
logger.Error(errMsg);
throw new IOException(errMsg);
}
FileInfo destFileInfo = new FileInfo(destFile);
destTemplateObjectTO.size = destFileInfo.Length.ToString();
JObject ansObj = Utils.CreateCloudStackObject(CloudStackTypes.TemplateObjectTO, destTemplateObjectTO);
newData = ansObj;
result = true;
}
else
{
details = "Data store combination not supported";
}
}
// Create volume from a template?
else if (srcTemplateObjectTO != null && destVolumeObjectTO != null)
{
// VolumeObjectTO guesses file extension based on existing files
// this can be wrong if the previous file had a different file type
var guessedDestFile = destVolumeObjectTO.FullFileName;
if (File.Exists(guessedDestFile))
{
logger.Info("Deleting existing file " + guessedDestFile);
File.Delete(guessedDestFile);
}
destVolumeObjectTO.format = srcTemplateObjectTO.format;
destFile = destVolumeObjectTO.FullFileName;
if (File.Exists(destFile))
{
logger.Info("Deleting existing file " + destFile);
File.Delete(destFile);
}
string srcFile = srcTemplateObjectTO.FullFileName;
if (!File.Exists(srcFile))
{
details = "Local template file missing from " + srcFile;
}
else
{
// TODO: thin provision instead of copying the full file.
File.Copy(srcFile, destFile);
destVolumeObjectTO.path = destVolumeObjectTO.uuid;
JObject ansObj = Utils.CreateCloudStackObject(CloudStackTypes.VolumeObjectTO, destVolumeObjectTO);
newData = ansObj;
result = true;
}
}
else if (srcVolumeObjectTO != null && destVolumeObjectTO != null)
{
var guessedDestFile = destVolumeObjectTO.FullFileName;
if (File.Exists(guessedDestFile))
{
logger.Info("Deleting existing file " + guessedDestFile);
File.Delete(guessedDestFile);
}
destVolumeObjectTO.format = srcVolumeObjectTO.format;
destFile = destVolumeObjectTO.FullFileName;
if (File.Exists(destFile))
{
logger.Info("Deleting existing file " + destFile);
File.Delete(destFile);
}
string srcFile = srcVolumeObjectTO.FullFileName;
if (!File.Exists(srcFile))
{
details = "Local template file missing from " + srcFile;
}
else
{
// Create the directory before copying the files. CreateDirectory
// doesn't do anything if the directory is already present.
Directory.CreateDirectory(Path.GetDirectoryName(destFile));
File.Copy(srcFile, destFile);
if (srcVolumeObjectTO.nfsDataStore != null && srcVolumeObjectTO.primaryDataStore == null)
{
logger.Info("Copied volume from secondary data store to primary. Path: " + destVolumeObjectTO.path);
}
else if (srcVolumeObjectTO.primaryDataStore != null && srcVolumeObjectTO.nfsDataStore == null)
{
destVolumeObjectTO.path = destVolumeObjectTO.path + "/" + destVolumeObjectTO.uuid;
if (destVolumeObjectTO.format != null)
{
destVolumeObjectTO.path += "." + destVolumeObjectTO.format.ToLower();
}
}
else
{
logger.Error("Destination volume path wasn't set. Unsupported source volume data store.");
}
// Create volumeto object deserialize and send it
JObject ansObj = Utils.CreateCloudStackObject(CloudStackTypes.VolumeObjectTO, destVolumeObjectTO);
newData = ansObj;
result = true;
}
}
else if (srcVolumeObjectTO != null && destTemplateObjectTO != null)
{
var guessedDestFile = destTemplateObjectTO.FullFileName;
if (File.Exists(guessedDestFile))
{
logger.Info("Deleting existing file " + guessedDestFile);
File.Delete(guessedDestFile);
}
destTemplateObjectTO.format = srcVolumeObjectTO.format;
destFile = destTemplateObjectTO.FullFileName;
if (File.Exists(destFile))
{
logger.Info("Deleting existing file " + destFile);
File.Delete(destFile);
}
string srcFile = srcVolumeObjectTO.FullFileName;
if (!File.Exists(srcFile))
{
details = "Local template file missing from " + srcFile;
}
else
{
// Create the directory before copying the files. CreateDirectory
// doesn't do anything if the directory is already present.
Directory.CreateDirectory(Path.GetDirectoryName(destFile));
File.Copy(srcFile, destFile);
FileInfo destFileInfo = new FileInfo(destFile);
// Write the template.properties file
PostCreateTemplate(Path.GetDirectoryName(destFile), destTemplateObjectTO.id, destTemplateObjectTO.name,
destFileInfo.Length.ToString(), srcVolumeObjectTO.size.ToString(), destTemplateObjectTO.format);
TemplateObjectTO destTemplateObject = new TemplateObjectTO();
destTemplateObject.size = srcVolumeObjectTO.size.ToString();
destTemplateObject.format = srcVolumeObjectTO.format;
destTemplateObject.path = destTemplateObjectTO.path + "/" + destTemplateObjectTO.uuid;
if (destTemplateObject.format != null)
{
destTemplateObject.path += "." + destTemplateObject.format.ToLower();
}
destTemplateObject.nfsDataStoreTO = destTemplateObjectTO.nfsDataStoreTO;
destTemplateObject.checksum = destTemplateObjectTO.checksum;
JObject ansObj = Utils.CreateCloudStackObject(CloudStackTypes.TemplateObjectTO, destTemplateObject);
newData = ansObj;
result = true;
}
}
else
{
details = "Data store combination not supported";
}
}
catch (Exception ex)
{
// Test by providing wrong key
details = CloudStackTypes.CopyCommand + " failed on exception, " + ex.Message;
logger.Error(details, ex);
}
object ansContent = new
{
result = result,
details = details,
newData = newData,
contextMap = contextMap
};
return ReturnCloudStackTypedJArray(ansContent, CloudStackTypes.CopyCmdAnswer);
}
}
