public IHttpActionResult Execute()
{
var fileJob = new FileJob(new SourceFolderScanner());
fileJob.Execute(null);
return(Ok());
}
public static void DoTest(long size, string password, string salt)
{
FileInfo source = null;
FileInfo pending = null;
FileInfo target = null;
FileInfo restored = null;
try
{
source = CreateRandomFile(size);
string sourceHash = KeepHasher.GetHash(source);
pending = new FileInfo(source.FullName + ".pending");
target = new FileInfo(source.FullName + ".target");
restored = new FileInfo(source.FullName + ".restored");
IFile testFile = new TestFile()
{
Name = source.Name,
Sha256 = sourceHash,
SizeBytes = size
};
FileJob job = new FileJob(testFile, source.FullName, pending.FullName, target.FullName, FileJob.ModeEnum.CompressEncrypt, password, salt);
job.Execute();
FileJob job2 = new FileJob(testFile, target.FullName, pending.FullName, restored.FullName, FileJob.ModeEnum.DecryptDecompress, password, salt);
job2.Execute();
string restoredHash = KeepHasher.GetHash(restored);
if (sourceHash != restoredHash)
{
throw new Exception();
}
}
finally
{
if (source != null && source.Exists)
{
source.Delete();
}
if (pending != null && pending.Exists)
{
pending.Delete();
}
if (target != null && target.Exists)
{
target.Delete();
}
if (restored != null && restored.Exists)
{
restored.Delete();
}
}
}
private void opnFileTimer_FileOk(object sender, CancelEventArgs e)
{
FileJob fj = new FileJob();
string[] str = new string[fj.ReadF(opnFileTimer.FileName).Length];
str = fj.ReadF(opnFileTimer.FileName);
tTimerList.Clear();
for (int i = 0; i < str.Length; i++)
{
tTimerList.AppendText(str[i] + Environment.NewLine);
}
}
public async Task <string> CreateFileJob(string cid)
{
using (var scope = GlobalServices.Container.BeginLifetimeScope())
{
var repository = scope.Resolve <ICloudSpeedRepository>();
var entity = new FileJob()
{
Cid = cid
};
await repository.CreateFileJob(entity);
await repository.Commit();
return(entity.Id);
}
}
public async Task TestGetFileQuote()
{
var text = "Hang down your head, Tom Dooley\nHang down your head and cry";
var raw = Encoding.UTF8.GetBytes(text);
var file = new FileJob("lyrics.txt", raw)
{
Slug = "test file",
SourceLanguage = "en",
TargetLanguage = "ja",
TranslationTier = TranslationTier.Standard
};
var quote = await client.Service.GetQuoteForFiles(file);
Assert.AreEqual(1, quote.Length);
Assert.AreEqual(text, quote[0].Body);
}
#pragma warning disable CS1572 // Комментарий XML имеет тег param для "name", но параметр с таким именем отсутствует.
/// <summary>
/// Exports the data to data base from directory C:\temp.
/// </summary>
/// <param name="view">The view.</param>
/// <param name="conDb">The con database.</param>
/// <param name="name">The name.</param>
public void ExportDataToDataBaseTemp(DataGridView view, string conDb)
#pragma warning restore CS1572 // Комментарий XML имеет тег param для "name", но параметр с таким именем отсутствует.
{
FileJob fileJ = new FileJob();
string[] filesname = Directory.GetFiles(@"C:\\temp\\");
for (int i = 0; i < filesname.Length; i++)
{
fileJ.Common_Import(filesname[i], view);
string[] names = filesname[i].Split(Convert.ToChar(92)); //devide string to array using as delimeter '\'
string[] s = names[names.Length - 1].Split(Convert.ToChar(46)); //devide string to array using as delimeter '.'
string[] s1 = s[0].Split('_'); //devide string to array using as delimeter '_'
//export to database
DBConn dBConn = new DBConn();
dBConn.DBCreateTableRandomMeas(view, conDb, s1[0]);
dBConn.DBExportDataCommon(view, conDb, s1[0]);
}
//copying files from C:\temp to subdirectory .\resolved
fileJ.CopyResolvedFiles();
}
private void btnTimer_Click(object sender, EventArgs e)
{
FileJob fj = new FileJob();
string[] str = new string[fj.ReadF("F:\\temp\\time.txt").Length];
str = fj.ReadF("F:\\temp\\time.txt");
tTimerList.Clear();
for (int i = 0; i < str.Length; i++)
{
tTimerList.AppendText(str[i] + Environment.NewLine);
}
str = new string[fj.ReadF("F:\\temp\\volt.txt").Length];
str = fj.ReadF("F:\\temp\\volt.txt");
tVoltageList.Clear();
for (int i = 0; i < str.Length; i++)
{
tVoltageList.AppendText(str[i] + Environment.NewLine);
}
}
public void ExportDataToDataBaseTempAll(DataGridView view, string conDB)
{
FileJob fj = new FileJob();
string[] filesname = Directory.GetFiles(@"C:\\temp\\");
//ProgressDB.Maximum = filesname.Length;
string name = "";
for (int i = 0; i < filesname.Length; i++)
{
fj.DNON_RNON_Import(filesname[i], view); //импорт файла
string[] names = filesname[i].Split(Convert.ToChar(92)); //разделение строки на массив по символу '\'
string[] s = names[names.Length - 1].Split(Convert.ToChar(46)); //разделение строки на массив по символу '.'
name = s[s.Length - 2].Replace("DNON", ""); //замена подстроки на ""
//кусок отвечающий за экспорт в базу данных
DBConn dBConn = new DBConn();
dBConn.DBCreateTableRandomMeas(view, conDB, name);
dBConn.DBExportDataRandom(view, conDB, name);
//пропуск фалов с RNON.txt
++i;
//ProgressDB.Value = i;
}
}
/// <summary>
/// Exports the SQL file.
/// </summary>
/// <param name="FileName">Name of the file.</param>
/// <param name="ConnectingString">The connecting string.</param>
/// <returns>boolean</returns>
public bool ExportSQLFile(string FileName, string ConnectingString)
{
FileJob fj = new FileJob();
string[] sql = fj.ReadF(FileName);
try
{
NpgsqlConnection pgcon = new NpgsqlConnection(ConnectingString);
pgcon.Open();
for (int i = 0; i < sql.Count(); i++)
{
NpgsqlCommand CSend = new NpgsqlCommand(sql[i], pgcon);
CSend.ExecuteNonQuery();
}
pgcon.Close();
return(true);
}
catch (Exception msg)
{
Console.WriteLine(msg.ToString());
return(false);
}
}
public async Task CreateFileJob(FileJob entity)
{
await DbContext.FileJobs.AddAsync(entity);
}
private void opnFileExcel_FileOk(object sender, CancelEventArgs e)
{
FileJob fj = new FileJob();
fj.LoadExcelFileToTxt(tVoltageList, tTimerList, opnFileExcel.FileName);
}
public FileJobDetailUI(FileJob job)
{
this.job = job;
InitializeComponent();
Initialize();
}
public void Analyze()
{
int errorCount = 0;
ILog log = Program.log;
log.Info("getting all files ... might take some time");
FileInfo[] allFiles = (from x in _Storage.PartitionManager.AllDirectories
from f in x.EnumerateFiles("*", SearchOption.AllDirectories)
select f).ToArray();
log.Info("done");
log.Info("parsing the sha256 from filename");
IEnumerable <FileInfo> files = allFiles.Where(x => x.Name.EndsWith(ObjectStorage.OBJ_EXTENSION, StringComparison.OrdinalIgnoreCase));
Dictionary <string, FileInfo> filesBySha256 = new Dictionary <string, FileInfo>();
foreach (FileInfo f in files)
{
string sha256 = f.Name.Substring(0, f.Name.Length - ObjectStorage.OBJ_EXTENSION.Length);
filesBySha256.Add(sha256, f);
}
log.Info("collecting all stored objects from manifest");
Dictionary <string, ManifestItem> manifestBySha256 = _Storage.Manifest.ManifestObjects.ToDictionary(x => x.Sha256source);
/// Phase 1
log.Info("check if all manifest files are present as files");
foreach (var x in manifestBySha256.Keys)
{
if (!filesBySha256.ContainsKey(x))
{
log.Error("missing file in storage, which is present in the manifest! " + x);
errorCount++;
}
}
/// Phase 2
log.Info("check if all files are present in the manifest");
foreach (var x in filesBySha256.Keys)
{
if (!manifestBySha256.ContainsKey(x))
{
log.Error("missing file in manifest, that was found in storage! " + x);
errorCount++;
}
}
/// Phase 3
log.Info("check size of all files with the manifest");
foreach (var hash in filesBySha256.Keys)
{
ManifestItem mf;
if (!manifestBySha256.TryGetValue(hash, out mf))
{
continue;
}
long size = mf.SizeTarget;
long size2 = filesBySha256[hash].Length;
if (size != size2)
{
log.Error("storage size mismatch! for " + hash);
errorCount++;
}
}
/// Phase 4
log.Info("look for pending files");
IEnumerable <FileInfo> pendingFiles = allFiles.Where(x => x.Name.EndsWith(ObjectStorage.PENDING_EXTENSION, StringComparison.OrdinalIgnoreCase));
foreach (var x in pendingFiles)
{
log.Error("found pending file " + x.FullName);
errorCount++;
}
/// Phase 5
log.Info("look for unknown files");
foreach (var x in allFiles)
{
if (!x.Name.EndsWith(ObjectStorage.OBJ_EXTENSION, StringComparison.OrdinalIgnoreCase) &&
!x.Name.EndsWith(ObjectStorage.PENDING_EXTENSION, StringComparison.OrdinalIgnoreCase))
{
log.Error("unknown file " + x.FullName);
errorCount++;
}
}
/// Phase 6
log.Info("look, if files from all inventories are present in manifest and nothing more");
FileInfo[] invFiles = _Storage.GetInventoryFiles();
HashSet <string> hashesFromAllInventories = new HashSet <string>();
foreach (FileInfo fi in invFiles)
{
log.Info(fi.FullName);
Inventory i = Inventory.FromXml(fi, _Configuration);
IList <string> shas = i.Folder.AllFiles.Select(x => x.Sha256).ToList();
foreach (string sha in shas)
{
hashesFromAllInventories.Add(sha);
if (!manifestBySha256.ContainsKey(sha))
{
log.Error("missing file in manifest from inventory " + fi.FullName + " " + sha);
errorCount++;
}
}
}
foreach (string sha in manifestBySha256.Keys)
{
if (!hashesFromAllInventories.Contains(sha))
{
log.Error("found file in manifest, which is in no inventory " + sha);
errorCount++;
}
}
ManifestItem[] ml = manifestBySha256.Values.ToArray();
int count = ml.Count();
if (!_Fast)
{
/// Phase 7
{
log.Info("test cecksums of all files in storage");
Action <ManifestItem> check = delegate(ManifestItem x)
{
string shaTarget = x.Sha256target;
FileInfo fi = new FileInfo(ObjectStorage.GetStorageTarget(x.Sha256source, _Storage.PartitionManager.GetDirByNumber(x.Partition)));
string shaFile = KeepHasher.GetHash(fi);
if (!string.Equals(shaTarget, shaFile, StringComparison.Ordinal))
{
log.Error("storage hash mismatch! for " + x.Sha256source);
Interlocked.Increment(ref errorCount);
}
};
if (ml.Length > 0)
{
ml.ParallelExecute(
x => x.SizeSource,
x => x.Sha256source,
x => x.Sha256target,
x => check(x),
6);
}
}
}
if (_Decompress)
{
// Phase 8 - SLOOWWW :-)
log.Info("actually decode file and check size and checksum");
for (int i = 0; i < count; i++)
{
double percentFiles = ((double)i / (double)count) * 100.0;
Program.log.Info(string.Format("file {0} / {1} ({2:0.00}%)", i + 1, count, percentFiles));
var x = ml[i];
log.Info("checking decompression and decryption for " + x.Sha256source);
FileInfo fi = new FileInfo(ObjectStorage.GetStorageTarget(x.Sha256source, _Storage.PartitionManager.GetDirByNumber(x.Partition)));
long sizeFile;
string shaFile = null;
using (Stream s = FileJob.JustDecryptAndDecompress(fi, _Configuration.Password, _Configuration.Salt))
{
sizeFile = s.Length;
shaFile = KeepHasher.GetHash(s);
}
if (!string.Equals(x.Sha256source, shaFile, StringComparison.Ordinal))
{
log.Error("decompressed and decrypted hash mismatch! for " + x.Sha256source);
errorCount++;
}
if (x.SizeSource != sizeFile)
{
log.Error("decompressed and decrypted size mismatch! for " + x.Sha256source);
errorCount++;
}
}
}
log.Info(errorCount + " errors !!");
}
/// <summary>
/// Dnons the rnon import.
/// </summary>
/// <param name="OpenFileDNON">The open file dnon.</param>
/// <param name="view">The view.</param>
public void DNON_RNON_Import(string OpenFileDNON, DataGridView view)
{
int cst = 0;
DataTable dt = new DataTable();
string fileN1 = OpenFileDNON;
string fileN2 = OpenFileDNON.Substring(0, OpenFileDNON.Length - 8) + "RNON.txt";
DataRow row;
FileJob FJ = new FileJob();
string[] s;
try
{
s = FJ.ReadF(OpenFileDNON);
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
return;
}
for (int i = 0; i < s.GetLength(0); i++)
{
ParseStringTab p = new ParseStringTab();
ParseStringTab p1 = new ParseStringTab();
p.AddString(s[i]);
//создание заголовков таблицы
if (i == 0)
{
p.AddString(s[i]);
IEnumerable <string> pdis = p.Distinct();
cst = pdis.Count();
foreach (string st in pdis)
{
p1.Add(st);
}
for (int j = 0; j < p1.Count; j++)
{
p1[j] = FJ.CorrectHead(p1[j]);
dt.Columns.Add(p1[j]);
}
}
else
{
//цикл для сортировки
int x = 0;
row = dt.NewRow();
for (int j = 0; j < p.Count(); j++)
{
try
{
row[x] = p[j];
++x;
if (x == cst)
{
dt.Rows.Add(row);
row = dt.NewRow();
x = 0;
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
}
p.Clear();
}
try
{
s = FJ.ReadF(fileN2);
}
catch (Exception ex)
{
ex.ToString();
}
for (int i = 1; i < s.GetLength(0); i++)
{
ParseStringTab p = new ParseStringTab();
ParseStringTab p1 = new ParseStringTab();
p.AddString(s[i]);
int x = 0;
row = dt.NewRow();
for (int j = 0; j < p.Count(); j++)
{
try
{
row[x] = p[j];
++x;
if (x == cst)
{
dt.Rows.Add(row);
row = dt.NewRow();
x = 0;
}
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
p.Clear();
}
view.DataSource = dt;
}
