public DataTable GenerateDataSGDT()
{
DataSGDT = new DataTable();
string initAddr = "0x00000000";
// Create Columns
DataSGDT.Columns.Add("Address", typeof(string));
DataSGDT.Columns.Add("Name", typeof(string));
DataSGDT.Columns.Add("Value", typeof(string));
foreach (var data in dxSG)
{
initAddr = data.Addr;
foreach (var value in data.StoredValue)
{
DataSGDT.Rows.Add(initAddr, data.Name.Remove(data.Name.Length - 1, 1), value);
int addr = Convert.ToInt32(Converter.ConvertHexToDec(initAddr.Remove(0, 2))) + 1;
initAddr = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex(addr.ToString()));
}
}
int limit = 2047 - Int32.Parse(Converter.ConvertHexToDec(initAddr.Remove(0, 2)));
for (int i = 0; i < limit; i++)
{
int addr = Convert.ToInt32(Converter.ConvertHexToDec(initAddr.Remove(0, 2))) + 1;
initAddr = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex(addr.ToString()));
DataSGDT.Rows.Add(initAddr, "", "0x00");
}
return(DataSGDT);
}
protected override void CleanData(object sender, ElapsedEventArgs eventArgs)
{
base.CleanData(sender, eventArgs);
var buildInBranches =
BuildRepository.Instance.Find(string.Format("WHERE StartDate IS NOT NULL AND StartDate < '{0}'",
(DateTime.Now - SchedulerInfo.SchedulePeriod).ToString("yyyy-MM-dd HH:mm:ss")))
.GroupBy(item => item.BranchId);
foreach (var builds in buildInBranches)
{
var buildsForRemoving = new List <Build>();
foreach (var build in builds)
{
// if build has any IsLastPassed screenshot - we cannot delete this build, because it contains in BaseLine
if (!ScreenShotRepository.Instance.FindByBuildId(build.Id)
.Any(screenshot => screenshot.IsLastPassed))
{
buildsForRemoving.Add(build);
}
}
buildsForRemoving.Skip(SchedulerInfo.HistoryItemsNumberToPreserve)
.Each(item => DataCleaner.DeleteObjectsTreeRecursively <Build>(item.Id, true));
}
}
private static List <Register> BNE()
{
string imm = tx.Params.Immediate;
string offset = imm.Substring(0, 1) + imm.Substring(11, 1) + imm.Substring(1, 10);
int num;
if (offset.Substring(0, 1) == "1")
{
offset = DataCleaner.PadHexOpValue(32, offset, '1');
num = Int32.Parse(Convert.ToString(Convert.ToInt32(offset, 2), 10));
}
else
{
num = Int32.Parse(Converter.ConvertBinToDec(offset));
}
int ioffSet = (num * 2) + Int32.Parse(Converter.ConvertHexToDec(tx.Address.Replace("0x", "")));
int paramOne = GetRegisterValue(rxSG, GetRegisterIndex(rxSG, tx.Params.RSourceOne));
int paramTwo = GetRegisterValue(rxSG, GetRegisterIndex(rxSG, tx.Params.RDestination));
if (paramOne != paramTwo)
{
NextAddr = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex(ioffSet.ToString()));
}
return(rxSG);
}
private static List <Register> LH()
{
// LW x1, var1
//string addr = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertBinToHex(tx.Params.Immediate));
//int index = dx.FindIndex(p => p.Addr == addr);
//string value = string.Empty;
//for (int i = 0; i < 2; i++)
//{
// value = dx[index].StoredValue[i] + value;
//}
//value = value.Replace("0x", "");
//rxSG[GetRegisterIndex(rxSG, tx.Params.RDestination)].Value = "0x" + DataCleaner.PadHexValue(8, value);
////rxSG[GetRegisterIndex(rxSG, tx.Params.RDestination)].Value = dx[index].Addr; ;
int imm = Int32.Parse(Converter.ConvertBinToDec(tx.Params.Immediate));
int sourceOne = 0;
if (tx.Params.RSourceOne != null)
{
sourceOne = Int32.Parse(Converter.ConvertHexToDec(
rxSG[GetRegisterIndex(rxSG, tx.Params.RSourceOne)].Value.Replace("0x", "")
));
}
string addr = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex((imm + sourceOne).ToString()));
string value = GetRowValue(addr, 2);
value = value.Replace("0x", "");
rxSG[GetRegisterIndex(rxSG, tx.Params.RDestination)].Value = "0x" + DataCleaner.PadHexValue(8, value);
return(rxSG);
}
public static Dictionary <Record, List <Record> > GetMatches(List <Record> records)
{
Dictionary <Record, List <Record> > ret = new Dictionary <Record, List <Record> >();
string[] finalDataSetList = File.ReadAllLines("c:/users/brush/desktop/finaldataset.csv");
DecisionTree[] forest = DataLoader.LoadForestFromDirectory("C:/users/brush/desktop/forest");
foreach (Record record in records)
{
ret.Add(record, new List <Record>());
}
Console.WriteLine("Searching for matches...");
//for (int c = 1; c < finalDataSetList.Length; c++)
Parallel.For(1, finalDataSetList.Length, c =>
{
if (c % 10000 == 0)
{
Console.WriteLine($"{(c / (finalDataSetList.Length * 1.0) * 100)}%");
}
string finalDataSetRow = finalDataSetList[c];
string[] bits = finalDataSetRow.Split(',');
if (bits[0] != "")
{
int enterpriseId = int.Parse(bits[0]);
if (enterpriseId > 15374761)
{
Record comparisonRecord = DataCleaner.CleanRecord(Record.FromFinalDatasetString(bits));
foreach (Record toMatch in records)
{
if (!toMatch.Equals(comparisonRecord))
{
RecordPair pair = new RecordPair
{
Record1 = toMatch,
Record2 = comparisonRecord,
};
bool isMatch = DecisionTreeBuilder.IsMatch(pair, forest, null);
if (isMatch)
{
lock (ret)
{
ret[toMatch].Add(comparisonRecord);
}
}
}
}
}
}
});
return(ret);
}
public void DeleteBranch(long id)
{
var builds = GetBuilds(id);
// Stop all builds
builds.Each(item => RunOperation("build", item.Id, "stop"));
DataCleaner.DeleteObjectsTreeRecursively <Branch>(id, true);
}
public virtual void OnDestroy()
{
m_redGroup.Clear();
m_blueGroup.Clear();
//PawnDataList.Clear();
m_pawnEntityList.Clear();
m_cleaner = null;
}
public MainForm(Client client)
{
InitializeComponent();
_client = client;
_client.GetInfoFromServer(mainDataGridView, lstTables);
DataCleaner.CleanDataGridView(mainDataGridView);
}
public static List <DataSegment> ValidateDS(List <string> code, out string err)
{
List <string> dsCode = DataCleaner.ExtractSegment(code, ".data", ".text");
List <DataSegment> dataSG = new List <DataSegment>();
dsCode = CheckFormat(code, dsCode, out err, out dataSG);
dataSG = SetStoredValue(dataSG);
dataSG = SetAddress(dataSG);
return(dataSG);
//return dsCode;
}
public void DeleteProject(long id)
{
var branches = GetBranches(id);
var builds = new List <long>();
branches.Each(item => builds.AddRange(item.Builds.Keys));
// Stop all builds
builds.Each(buildId => RunOperation("build", buildId, "stop"));
DataCleaner.DeleteObjectsTreeRecursively <Project>(id, true);
}
// REGISTER
private static List <Register> ADD()
{
// ADD x1, x2, x3
int rsOne = GetRegisterValue(rxSG, GetRegisterIndex(rxSG, tx.Params.RSourceOne));
int rsTwo = GetRegisterValue(rxSG, GetRegisterIndex(rxSG, tx.Params.RSourceTwo));
string rsDest = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex((rsOne + rsTwo).ToString()));
int index = GetRegisterIndex(rxSG, tx.Params.RDestination);
rxSG[index].Value = rsDest;
return(rxSG);
}
public void Given_officeName_Should_clean_the_name_to_make_it_uniform(string officeName, string expected)
{
//Arrange
var dataCleaner = new DataCleaner();
//Act
var actual = dataCleaner.CleanData(officeName);
//Assert
Assert.AreEqual(expected, actual);
}
private static List <TextSegment> SetAddress(List <TextSegment> txSegment)
{
int i = 4096;
foreach (var tx in txSegment)
{
tx.Address = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex(i.ToString()));
i += 4;;
}
return(txSegment);
}
public static void Test()
{
Console.WriteLine("Line 1:");
string line1 = Console.ReadLine();
Console.WriteLine("Line 2:");
string line2 = Console.ReadLine();
RecordPair pair = new RecordPair();
pair.Record1 = DataCleaner.CleanRecord(Record.FromString(line1));
pair.Record2 = DataCleaner.CleanRecord(Record.FromString(line2));
DecisionTree[] forest = DataLoader.LoadForestFromDirectory(".");
TreeLogger logger = new TreeLogger();
bool isMatch = DecisionTreeBuilder.IsMatch(pair, forest, logger);
Console.WriteLine("Search for those in training data who make it there?");
string response = Console.ReadLine();
//RecordPair pair2 = new RecordPair();
//pair2.Record1 = Record.FromString("LAZAR,,KALLER,,M,,16/10/1965,,,,,,,-1,,,4839002,15479245,");
//pair2.Record2 = Record.FromString("ADRIENNE,,KELLEY,,F,895535860,16/10/1965,9175738850,,1560 SILVER ST,2H,BRONX,NY,10461,,[email protected],4799491,15637549,");
//bool ret = DecisionTreeBuilder.ReplayDecision(pair2, logger.SplittingQuestionsToTheBottom);
if (response.ToLower() == "y")
{
using (StreamWriter sw = File.AppendText("c:/users/brush/desktop/gothere.txt"))
{
List <RecordPair> pairs = new List <RecordPair>();
Console.Write("Loading training data for this iteration...");
pairs.AddRange(DataLoader.GetPositivesFromMRNData("mrns.csv"));
pairs.AddRange(DataLoader.GetHandPassedSets("more.csv"));
pairs.AddRange(DataLoader.GetRejectedRecordPairs("rejected.txt"));
//pairs.AddRange(DataLoader.GetPairsFromMisfitsFile("misfits.txt"));
Console.WriteLine("...done");
Parallel.ForEach(pairs, p =>
{
if (DecisionTreeBuilder.ReplayDecision(p, logger.SplittingQuestionsToTheBottom))
{
lock (sw)
{
sw.WriteLine(p);
}
}
});
}
}
return;
}
private void btnAccept_Click(object sender, EventArgs e)
{
if (txtNewValue.Text == "")
{
Close();
return;
}
_client.UpdateTable(_dataGridView, _lstBox, txtNewValue.Text);
DataCleaner.CleanDataGridView(_dataGridView);
Close();
}
private void CheckDataBT_Click(object sender, RoutedEventArgs e)
{
DataCleaner dataCleaner = new DataCleaner(dataSet, Standard);
MessageBox.Show("Program is cleaning the Data, please wait a few seconds...");
// Console.ReadLine();
dataCleaner.Clean();
dataSet = dataCleaner.dataSet;
DataDG.DataContext = dataSet;
DataDG.UpdateLayout();
System.Windows.MessageBox.Show(@"Data has been cleaned");
}
public int GetRestorationTime(double size)
{
int v = DataCleaner.GetNearstFifty(Convert.ToInt32(size));
while (!Data.ContainsKey(v))
{
v -= 50;
}
int rtime = Convert.ToInt32(Data[v].GetAverageCheckpointValue().RestorationTime *Global.Second);
return(rtime);
}
public void TestDisabledCleaner()
{
var logCleaner = new Mock <ILogCleaner>();
var cleaner = new DataCleaner(logCleaner.Object);
var settings = new Setting()
{
DeleteOldLogs = false, OldLogDeleteDays = 20
};
cleaner.Initialize(settings);
logCleaner.Verify(l => l.Clean(), Times.Never());
}
//public static List<Register> ExecuteOperation(TextSegment tx, List<DataSegment> dx, List<Register> rxSG)
public static List <Register> ExecuteOperation(TextSegment _tx, DataTable _dx, List <Register> _rxSG)
{
if (_tx.Address == NextAddr)
{
tx = _tx;
dxDT = _dx;
rxSG = _rxSG;
if (tx.Params.Immediate == null)
{
tx.Params.Immediate = "000000";
}
string next = (Int32.Parse(Converter.ConvertHexToDec(NextAddr.Replace("0x", ""))) + 4).ToString();
NextAddr = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex(next));
switch (tx.Operation.ToUpper())
{
case "LW": rxSG = LW(); break;
case "LH": rxSG = LH(); break;
case "LB": rxSG = LB(); break;
case "ADD": rxSG = ADD(); break;
case "SLT": rxSG = SLT(); break;
case "SUB": rxSG = SUB(); break;
case "ADDI": rxSG = ADDI(); break;
case "SLTI": rxSG = SLTI(); break;
case "SW": rxSG = SW(); break;
case "SH": rxSG = SH(); break;
case "SB": rxSG = SB(); break;
case "BEQ": rxSG = BEQ(); break;
case "BNE": rxSG = BNE(); break;
case "BLT": rxSG = BLT(); break;
case "BGE": rxSG = BGE(); break;
}
}
return(rxSG);
}
public AccountsController(ApplicationDbContext db, UserManager <AppUser> userManager,
SignInManager <AppUser> signInManager, RoleManager <IdentityRole> roleManager, DataCleaner dataCleaner,
IOptions <AppSettings> appSettings, IEmailSender emailsender, ExceptionsLoggerService exceptionLogger)
{
_db = db;
_userManager = userManager;
_signManager = signInManager;
_roleManager = roleManager;
_appSettings = appSettings.Value;
_emailsender = emailsender;
_exceptionLogger = exceptionLogger;
_dataCleaner = dataCleaner;
}
public static List <Record> GetCleanedRecordsFromMRNFile(string mrnFilePath)
{
string[] lines = File.ReadAllLines(mrnFilePath);
List <Record> allRecords = new List <Record>();
for (int c = 0; c < lines.Length; c += 3)
{
allRecords.Add(Record.FromString(lines[c]));
allRecords.Add(Record.FromString(lines[c + 1]));
}
return(DataCleaner.CleanRecordPairs(allRecords));
}
private static List <Register> BGE()
{
string imm = tx.Params.Immediate;
string offset = imm.Substring(0, 1) + imm.Substring(11, 1) + imm.Substring(1, 9);
int ioffSet = (Int32.Parse(Converter.ConvertBinToDec(offset)) * 2) + Int32.Parse(Converter.ConvertHexToDec(tx.Address.Replace("0x", "")));
int paramOne = GetRegisterValue(rxSG, GetRegisterIndex(rxSG, tx.Params.RSourceOne));
int paramTwo = GetRegisterValue(rxSG, GetRegisterIndex(rxSG, tx.Params.RDestination));
if (paramTwo > paramOne)
{
NextAddr = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex(ioffSet.ToString()));
}
return(rxSG);
}
private static string CheckValues(List <string> storedValue, out string err)
{
err = string.Empty;
List <string> cleanedVal = new List <string>();
if (storedValue.Count == 0)
{
err = " \t- Please initialize variable." + "\r\n";
return(string.Empty);
}
foreach (var sv in storedValue)
{
string tmp = sv;
//if (tmp.Length > 3)
//{
// if (tmp.Substring(0, 2) == "0x")
// tmp = sv.Remove(0, 2);
//}
tmp = tmp.Replace("0x", "");
NUM_TYPES type = DataCleaner.CheckNumberType(tmp);
if ((type == NUM_TYPES.DEC) || (type == NUM_TYPES.HEX) || (type == NUM_TYPES.BIN))
{
if (sv.StartsWith("0x"))
{
cleanedVal.Add(tmp);
}
else
{
cleanedVal.Add(Converter.ConvertToHex(tmp));
}
}
else
{
if (DataCleaner.IsNumType(tmp, @"\A[-]{0,1}$?[0-9]+\Z"))
{
string binValue = Converter.ConvertDecToBin(tmp);
binValue = binValue.Substring(binValue.Length - 32, 32);
cleanedVal.Add(Converter.ConvertBinToHex(binValue));
}
else
{
err = " \t- Specified an invalid value (" + tmp + ") that is neither an INT nor HEX." + "\r\n";
}
}
}
return(string.Join(",", cleanedVal));
}
private void lstTables_SelectedIndexChanged(object sender, EventArgs e)
{
if (lstTables.SelectedIndex == -1)
{
return;
}
btnUpdateTable.Enabled = true;
lstQueries.ClearSelected();
lstTags.ClearSelected();
_client.SelectTable(mainDataGridView, lstTables);
DataCleaner.CleanDataGridView(mainDataGridView);
}
private static string GetRowValue(string addr, int limit)
{
int i = -1;
int oLimit = limit;
string value = string.Empty;
foreach (DataRow row in dxDT.Rows)
{
if (oLimit != limit)
{
value = (string)row["Value"] + value.Replace("0x", "");
limit--;
}
if ((string)row["Address"] == addr)
{
value = (string)row["Value"] + value;
limit--;
}
if (limit == 0)
{
break;
}
i++;
}
if (!string.IsNullOrEmpty(value))
{
value = value.Replace("0x", "");
int bigE = Int32.Parse(Converter.ConvertHexToDec(value.Substring(0, 1)));
if (bigE < 8)
{
value = DataCleaner.PadHexOpValue(8, value, '0');
}
else
{
value = DataCleaner.PadHexOpValue(8, value, 'F');
}
}
return(value);
}
public static List <string> ValidateTS(DataTable _dataDT, List <DataSegment> dataSG, List <string> code, out string err, out List <TextSegment> txSegment)
{
dataDT = _dataDT;
dxSG = dataSG;
err = string.Empty;
string msg = string.Empty;
List <string> txCode = DataCleaner.ExtractSegment(code, ".text", string.Empty);
txSegment = new List <TextSegment>();
txCode = CheckFormat(txCode, code, out err, out txSegment);
if (string.IsNullOrEmpty(err))
{
txSegment = SetAddress(txSegment);
txSegment = OpCodeController.GenerateOpCode(txSegment);
}
return(txCode);
}
// IMMEDIATE
private static List <Register> ADDI()
{
// ADDI x1, x2, x3
int rsOne = GetRegisterValue(rxSG, GetRegisterIndex(rxSG, tx.Params.RSourceOne));
string immRsTwo = tx.Params.Immediate + tx.Params.RSourceTwo;
if (immRsTwo.Substring(0, 1) == "1")
{
immRsTwo = DataCleaner.PadHexOpValue(64, immRsTwo, '1');
}
int rsTwo = Int32.Parse(Converter.ConvertBinToDec(immRsTwo));
string rsDest = "0x" + DataCleaner.PadHexValue(8, Converter.ConvertDecToHex((rsOne + rsTwo).ToString()));
int index = GetRegisterIndex(rxSG, tx.Params.RDestination);
rxSG[index].Value = rsDest;
return(rxSG);
}
public static List <RecordPair> LoadTrainingDataFromNoHomoFile(string noHomoFilePath)
{
List <RecordPair> ret = new List <RecordPair>();
string[] lines = File.ReadAllLines(noHomoFilePath);
for (int c = 0; c < lines.Length; c += 4)
{
RecordPair pair = new RecordPair();
pair.IsMatch = bool.Parse(lines[c]);
pair.Record1 = DataCleaner.CleanRecord(Record.FromString(lines[c + 1]));
pair.Record2 = DataCleaner.CleanRecord(Record.FromString(lines[c + 2]));
ret.Add(pair);
}
return(ret);
}
public static void Main(string[] args)
{
IFileSystemProvider fileSystemProvider = new FileSystemProvider();
IDataCleaner dataCleaner = new DataCleaner();
var nettCalculator = new NettCalculatorUseCase(fileSystemProvider, dataCleaner);
Console.WriteLine("Enter Office to calculate nett profit for: ");
var office = Console.ReadLine();
var nettProfit = nettCalculator.CalculateNettProfit("Question 1 input.csv", office);
Console.WriteLine("Nett Profit: " + nettProfit);
Console.WriteLine("Press Enter to see the office with the largest nett profit");
Console.ReadKey();
var officeWitMaxnettProfit = nettCalculator.FindLargestNettProfit("Question 2 input.csv");
Console.WriteLine(officeWitMaxnettProfit);
Console.ReadKey();
}
static async Task Main(string[] args)
{
Parser parser = new Parser();
DataCleaner dc = new DataCleaner();
Console.WriteLine("Enter URL of web page:");
string url = Console.ReadLine();
string rawHTMLData = parser.GetHTMLCode(url);
string cleanData = dc.CleanData(rawHTMLData);
FileWorker.WriteToFile("raw_parsed_data.txt", rawHTMLData);
FileWorker.WriteToFile("clean_parsed_data.txt", cleanData);
Console.WriteLine("Done!");
string pathToDataFile = Environment.CurrentDirectory;
Console.WriteLine($"Parsed data there: {pathToDataFile}");
Console.ReadKey();
}
