static void MongoInsertFromZones()
{
try
{
List <TempZoneID> temp1 = DtToList <TempZoneID> .ConvertToModel(MySQLHelper.Query("select id as zoneid from Temp where type in (1,4)").Tables[0]);
DataTable dtid = MySQLHelper.Query("select min(id),max(id) from zones").Tables[0];
long min = Convert.ToInt32(dtid.Rows[0][0]);
long max = Convert.ToInt32(dtid.Rows[0][1]);
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();//开始计时
long index = min;
do
{
DataTable dt = MySQLHelper.Query("SELECT z.id,z.zone,z.userid,CASE when d.maxfensheng<1 then 0 else 1 end as level,z.nsstate,z.active,z.forcestop from zones as z left join domainlevel as d on z.DomainLevel=d.levelvalue where z.userid<>348672 and z.id between " + index + " and " + (index + 20000) + " and z.Active='Y' and z.ForceStop='N'").Tables[0];
Console.WriteLine("GetDataTabel from mysql Use Time={0};", watch.ElapsedMilliseconds);
List <zones> zonesList = DtToList <zones> .ConvertToModel(dt);
Console.WriteLine("DataTable Convert to ModelList; Use time={0};", watch.ElapsedMilliseconds);
List <ZonesSimple> dl = new List <ZonesSimple>();
foreach (zones z in zonesList)
{
if (temp1.FindAll(tz => tz.zoneid == z.id).Count == 0)
{
dl.Add(Row2ZoneSimple(z));
}
}
Console.WriteLine("Data Filter; Use time={0};", watch.ElapsedMilliseconds);
var client = DriverConfiguration.Client;
var db = client.GetDatabase(DriverConfiguration.DatabaseNamespace.DatabaseName);
IMongoCollection <ZonesSimple> categories = db.GetCollection <ZonesSimple>("zones");
if (dl.Count > 0)
{
try
{
categories.InsertMany(dl);
}
catch (Exception ex) {
Console.WriteLine(ex.Message);
}
}
dl.Clear();
Console.WriteLine("MongoDB Inserted; Use time={0};", watch.ElapsedMilliseconds);
index = index + 20001;
Console.WriteLine("min={0};max={1};index={2};use time={3}", min, max, index, watch.ElapsedMilliseconds);
Console.WriteLine("==============================================");
} while (index < max);
Console.WriteLine("End min={0};max={1};index={2};use time {3}", min, max, index, watch.ElapsedMilliseconds);
watch.Stop();//停止计时
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
static void CheckMXData()
{
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();//开始计时
var client = DriverConfiguration.Client;
var db = client.GetDatabase(DriverConfiguration.DatabaseNamespace.DatabaseName);
for (int i = 0; i < 16; i++)
{
IMongoCollection <DnsRecordsSimple> collection = db.GetCollection <DnsRecordsSimple>(i.ToString("x"));
var builder = Builders <DnsRecordsSimple> .Filter;
var drslist = collection.Find(builder.And(builder.Eq("type", "MX"))).ToList <DnsRecordsSimple>();
string ids = "";
for (int j = 0; j < drslist.Count; j++)
{
if (j > 0 && j % 10 == 9)
{
ids = ids + drslist[j].rid.ToString();
DataTable dt = MySQLHelper.Query("select d.id,d.zoneid,d.zone,d.host,d.type,d.data,d.ttl,d.view,d.mx_priority,d.userid from dnsrecords as d where d.id in(" + ids + ")").Tables[0];
List <dnsrecords> rListtemp = DtToList <dnsrecords> .ConvertToModel(dt);
if (dt.Rows.Count > 0)
{
for (int k = j; k > j - 10; k--)
{
var tmepRecord = rListtemp.Find(s => s.id.Equals(drslist[k].rid));
if (tmepRecord != null && drslist[k].rdata != tmepRecord.mx_priority.ToString() + " " + tmepRecord.data.ToString().ToLower())
{
Console.WriteLine(drslist[k].domain + " " + drslist[k].rdata);
Console.WriteLine(tmepRecord.mx_priority.ToString() + " " + tmepRecord.data.ToString().ToLower());
Console.WriteLine("========================================");
}
}
}
ids = "";
}
else
{
ids = ids + drslist[j].rid.ToString() + ",";
}
}
Console.WriteLine(i.ToString("x") + " Collection Processed; Use Time=" + watch.ElapsedMilliseconds);
}
}
static void MongoInsertFromZones2()
{
try
{
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();//开始计时
DataTable dt = MySQLHelper.Query("SELECT * from zones where userid=424920").Tables[0];
Console.WriteLine("GetDataTabel from mysql Use Time={0};", watch.ElapsedMilliseconds);
List <Zones> zonesList = DtToList <Zones> .ConvertToModel(dt);
Console.WriteLine("DataTable Convert to ModelList; Use time={0};", watch.ElapsedMilliseconds);
var client = DriverConfiguration.Client;
var db = client.GetDatabase(DriverConfiguration.DatabaseNamespace.DatabaseName);
IMongoCollection <Zones> categories = db.GetCollection <Zones>("ZonesEntirety");
foreach (Zones z in zonesList)
{
z._id = Utility.StringHelper.CalculateMD5Hash(z.Zone).ToLower();
}
try
{
categories.InsertMany(zonesList);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.WriteLine("MongoDB Inserted; Use time={0};", watch.ElapsedMilliseconds);
watch.Stop();//停止计时
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
static void RefreshOldData()
{
try
{
List <TempZoneID> temp1 = DtToList <TempZoneID> .ConvertToModel(MySQLHelper.Query("select id as zoneid from Temp where type in (1,4)").Tables[0]);
DataTable dtid = MySQLHelper.Query("select min(id),max(id) from zones").Tables[0];
long min = Convert.ToInt32(dtid.Rows[0][0]);
long max = Convert.ToInt32(dtid.Rows[0][1]);
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();//开始计时
long index = min;
do
{
DataTable dt = MySQLHelper.Query("SELECT z.id,z.zone,z.userid,CASE when d.maxfensheng<1 then 0 else 1 end as level,z.nsstate,z.active,z.forcestop from zones as z left join domainlevel as d on z.DomainLevel=d.levelvalue where z.userid<>348672 and z.id between " + index + " and " + (index + 10000) + " and z.Active='Y' and z.ForceStop='N'").Tables[0];
Console.WriteLine("GetDataTabel from mysql Use Time={0};", watch.ElapsedMilliseconds);
List <zones> zonesList = DtToList <zones> .ConvertToModel(dt);
Console.WriteLine("DataTable Convert to ModelList; Use time={0};", watch.ElapsedMilliseconds);
List <ZonesSimple> dl = new List <ZonesSimple>();
foreach (zones z in zonesList)
{
if (temp1.FindAll(tz => tz.zoneid == z.id).Count == 0)
{
dl.Add(Row2ZoneSimple(z));
}
}
Console.WriteLine("Data Filter; Use time={0};", watch.ElapsedMilliseconds);
var client = DriverConfiguration.Client;
var db = client.GetDatabase(DriverConfiguration.DatabaseNamespace.DatabaseName);
IMongoCollection <ZonesSimple> categories = db.GetCollection <ZonesSimple>("zones");
int processCount = 0;
IList <string> zoneArry = new List <string>();
List <ZonesSimple> tempzl = new List <ZonesSimple>();
foreach (ZonesSimple zs in dl)
{
zoneArry.Add(zs.domain);
tempzl.Add(zs);
if ((processCount > 0 && processCount % 200 == 0) || processCount == dl.Count - 1)
{
List <ZonesSimple> mzl = categories.Find(Builders <ZonesSimple> .Filter.In("domain", zoneArry)).ToList <ZonesSimple>();
foreach (ZonesSimple z in tempzl)
{
var tz = mzl.SingleOrDefault(s => s.domain == z.domain);
if (tz != null)
{
if (tz.is_stop != z.is_stop || tz.force_stop != z.force_stop)
{
var update = Builders <ZonesSimple> .Update.Set("is_stop", z.is_stop).Set("force_stop", z.force_stop);
categories.UpdateOne(Builders <ZonesSimple> .Filter.Eq("domain", z.domain), update);
Console.WriteLine("Update Domain " + z.domain);
}
}
else
{
categories.InsertOne(z);
Console.WriteLine("Insert Domain " + z.domain);
}
}
zoneArry.Clear();
tempzl.Clear();
Console.WriteLine("Process Count {0} Use Time {1}", processCount, watch.ElapsedMilliseconds);
}
processCount++;
}
dl.Clear();
index = index + 10001;
Console.WriteLine("min={0};max={1};index={2};use time={3}", min, max, index, watch.ElapsedMilliseconds);
Console.WriteLine("==============================================");
} while (index < max);
Console.WriteLine("End min={0};max={1};index={2};use time {3}", min, max, index, watch.ElapsedMilliseconds);
watch.Stop();//停止计时
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
static void RefreshNewColumn()
{
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();//开始计时
List <ZonesSimple> dl = new List <ZonesSimple>();
List <AuthoritiesSimple>[] ala = new List <AuthoritiesSimple>[16] {
new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>()
};
List <DnsRecordsSimple>[] dla = new List <DnsRecordsSimple>[16] {
new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>()
};
var client = DriverConfiguration.Client;
var db = client.GetDatabase(DriverConfiguration.DatabaseNamespace.DatabaseName);
IMongoCollection <ZonesSimple> cz = db.GetCollection <ZonesSimple>("zones");
DataTable dt = MySQLHelper.Query("SELECT z.id,z.zone,z.userid,CASE when d.maxfensheng<1 then 0 else 1 end as level,z.nsstate,z.active,z.forcestop from zones as z left join domainlevel as d on z.DomainLevel=d.levelvalue where z.userid<>348672 and (z.Active='N' or z.ForceStop='Y' ) ").Tables[0];
DataTable dta = MySQLHelper.Query("select a.id,a.zone,host,data,type,ttl,mbox,serial,refresh,retry,expire,minimum ,t.userid,a.zoneid from authorities as a left join zones as t on a.ZoneID=t.id where t.userid<>348672 and (t.Active='N' or t.ForceStop='Y')").Tables[0];
DataTable dtd = MySQLHelper.Query("select d.id,d.zoneid,d.zone,d.host,d.type,d.data,d.ttl,d.view,d.mx_priority,d.userid,d.active from dnsrecords as d left join zones as t on d.ZoneID=t.id where t.userid<>348672 and (t.Active='N' or t.ForceStop='Y')").Tables[0];
Console.WriteLine("zones total count " + dt.Rows.Count);
Console.WriteLine("authorities total count " + dta.Rows.Count);
Console.WriteLine("dnsrecords total count " + dtd.Rows.Count);
List <zones> zonesList = DtToList <zones> .ConvertToModel(dt);
int deleteCount = 0;
IList <string> zoneArry = new List <string>();
foreach (zones z in zonesList)
{
zoneArry.Add(z.zone + ".");
ZonesSimple zs = Row2ZoneSimple(z);
dl.Add(zs);
if ((deleteCount > 0 && deleteCount % 200 == 0) || deleteCount == zonesList.Count)
{
cz.DeleteMany(Builders <ZonesSimple> .Filter.In("domain", zoneArry));
string rrcol = StringHelper.CalculateMD5Hash(z.zone + ".").Substring(0, 1);
IMongoCollection <DnsRecordsSimple> cd = db.GetCollection <DnsRecordsSimple>(rrcol);
cd.DeleteMany(Builders <DnsRecordsSimple> .Filter.In("domain", zoneArry));
Console.WriteLine("delete count " + deleteCount + " use time " + watch.ElapsedMilliseconds);
if (dl.Count > 0)
{
cz.InsertMany(dl);
Console.WriteLine("insert zones use time " + watch.ElapsedMilliseconds);
}
zoneArry.Clear();
dl.Clear();
}
deleteCount++;
}
Console.WriteLine("delete complete use time " + watch.ElapsedMilliseconds);
List <authorities> aList = DtToList <authorities> .ConvertToModel(dta);
List <authorities> drl = new List <authorities>();
string domain = "";
for (int idx = 0; idx < aList.Count;)
{
domain = aList[idx].zone.ToString().ToLower() + ".";
string collectionname = StringHelper.CalculateMD5Hash(domain).ToLower().Substring(0, 1);
drl.Add(aList[idx]);
while (idx < (aList.Count - 1) && aList[idx].zone == aList[idx + 1].zone)
{
drl.Add(aList[idx + 1]);
idx++;
}
if (drl.Count > 1)
{
foreach (AuthoritiesSimple a in Row2Authorities(drl))
{
ala[Int32.Parse(collectionname, System.Globalization.NumberStyles.HexNumber)].Add(a);
}
}
drl.Clear();
domain = "";
idx++;
}
List <dnsrecords> rList = DtToList <dnsrecords> .ConvertToModel(dtd);
List <dnsrecords> unList = new List <dnsrecords>();
foreach (dnsrecords dr in rList)
{
if (CheckRecordHost(dr.host, dr.type) && CheckRecordData(dr.data, dr.type, dr.view, dr.host))
{
DnsRecordsSimple d = Row2DnsRecords(dr);
string collectionname = StringHelper.CalculateMD5Hash(d.domain).ToLower().Substring(0, 1);
int idx = Int32.Parse(collectionname, System.Globalization.NumberStyles.HexNumber);
dla[idx].Add(d);
}
else
{
unList.Add(dr);
}
}
Console.WriteLine("start insert use time " + watch.ElapsedMilliseconds);
for (int i = 0; i < 16; i++)
{
IMongoCollection <DnsRecordsSimple> collection = db.GetCollection <DnsRecordsSimple>(i.ToString("x"));
if (ala[i].Count > 0)
{
collection.InsertMany(ala[i]);
}
if (dla[i].Count > 0)
{
collection.InsertMany(dla[i]);
}
}
if (unList.Count > 0)
{
Console.WriteLine("uninsert dnsrecords " + unList.Count);
}
Console.WriteLine("mission complete use time " + watch.ElapsedMilliseconds);
}
static void MongoInsertFromDnsrecords()
{
try
{
List <TempZoneID> temp = DtToList <TempZoneID> .ConvertToModel(MySQLHelper.Query("select id as zoneid from Temp").Tables[0]);
DataTable dtid = MySQLHelper.Query("select min(id),max(id) from zones").Tables[0];
long min = Convert.ToInt32(dtid.Rows[0][0]);
long max = Convert.ToInt32(dtid.Rows[0][1]);
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();//开始计时
long index = min;
do
{
DataTable dt = MySQLHelper.Query("select d.id,d.zoneid,d.zone,d.host,d.type,d.data,d.ttl,d.view,d.mx_priority,d.userid,d.active from dnsrecords as d left join zones as t on d.ZoneID=t.id where d.Active='Y' and d.type<>'PTR' and d.ZoneID BETWEEN " + index + " and " + (index + 20000) + " and t.userid<>348672 and t.id is not NULL and t.Active='Y' and t.ForceStop='N' order by d.zone").Tables[0];
Console.WriteLine("GetDataTabel from mysql Use Time={0};", watch.ElapsedMilliseconds);
List <dnsrecords> rListtemp = DtToList <dnsrecords> .ConvertToModel(dt);
Console.WriteLine("DataTable Convert to ModelList; Use time={0};", watch.ElapsedMilliseconds);
List <dnsrecords> rList = new List <dnsrecords>();
foreach (dnsrecords d in rListtemp)
{
if (temp.FindAll(tz => tz.zoneid == d.zoneid).Count == 0)
{
rList.Add(d);
}
}
List <dnsrecords> unList = new List <dnsrecords>();
List <DnsRecordsSimple>[] dla = new List <DnsRecordsSimple>[16] {
new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>(), new List <DnsRecordsSimple>()
};
foreach (dnsrecords dr in rList)
{
if (CheckRecordHost(dr.host, dr.type) && CheckRecordData(dr.data, dr.type, dr.view, dr.host))
{
DnsRecordsSimple d = Row2DnsRecords(dr);
string collectionname = StringHelper.CalculateMD5Hash(d.domain).ToLower().Substring(0, 1);
int idx = Int32.Parse(collectionname, System.Globalization.NumberStyles.HexNumber);
dla[idx].Add(d);
}
else
{
unList.Add(dr);
}
}
Console.WriteLine("ModelList Convert and Filter; Use time={0};", watch.ElapsedMilliseconds);
var client = DriverConfiguration.Client;
var db = client.GetDatabase(DriverConfiguration.DatabaseNamespace.DatabaseName);
for (int i = 0; i < 16; i++)
{
IMongoCollection <DnsRecordsSimple> collection = db.GetCollection <DnsRecordsSimple>(i.ToString("x"));
if (dla[i].Count > 0)
{
collection.InsertMany(dla[i]);
}
}
Console.WriteLine("MongoDB Inserted; Use time={0};", watch.ElapsedMilliseconds);
if (unList.Count > 0)
{
IMongoCollection <dnsrecords> collection2 = db.GetCollection <dnsrecords>("dnsrecordswrong");
collection2.InsertMany(unList);
}
Console.WriteLine("MongoDB Inserted Wrong Data; Use time={0};", watch.ElapsedMilliseconds);
index = index + 20001;
Console.WriteLine("min={0};max={1};index={2};use time {3}", min, max, index, watch.ElapsedMilliseconds);
Console.WriteLine("==============================================");
} while (index < max);
Console.WriteLine("End min={0};max={1};index={2};use time {3}", min, max, index, watch.ElapsedMilliseconds);
watch.Stop();//停止计时
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
static void MongoInsertFromAuthorities()
{
try
{
List <TempZoneID> temp = DtToList <TempZoneID> .ConvertToModel(MySQLHelper.Query("select id as zoneid from Temp").Tables[0]);
DataTable dtid = MySQLHelper.Query("select min(id),max(id) from zones").Tables[0];
long min = Convert.ToInt32(dtid.Rows[0][0]);
long max = Convert.ToInt32(dtid.Rows[0][1]);
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();//开始计时
long index = min;
do
{
DataTable dt = MySQLHelper.Query("select a.id,a.zone,host,data,type,ttl,mbox,serial,refresh,retry,expire,minimum ,t.userid,a.zoneid from authorities as a left join zones as t on a.ZoneID=t.id where a.ZoneID BETWEEN " + index + " and " + (index + 20000) + " and t.userid<>348672 and t.id is not NULL and t.Active='Y' and t.ForceStop='N' order by a.zone,a.type").Tables[0];
Console.WriteLine("getdatatabel from mysql use time={0};", watch.ElapsedMilliseconds);
List <authorities> aListtemp = DtToList <authorities> .ConvertToModel(dt);
Console.WriteLine("datatable convert to modellist; use time={0};", watch.ElapsedMilliseconds);
List <authorities> aList = new List <authorities>();
foreach (authorities a in aListtemp)
{
if (temp.FindAll(tz => tz.zoneid == a.zoneid).Count == 0)
{
aList.Add(a);
}
}
List <AuthoritiesSimple>[] ala = new List <AuthoritiesSimple>[16] {
new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>(), new List <AuthoritiesSimple>()
};
List <authorities> drl = new List <authorities>();
List <AuthoritiesSimple> dl = new List <AuthoritiesSimple>();
string domain = "";
for (int idx = 0; idx < aList.Count;)
{
domain = aList[idx].zone.ToString().ToLower() + ".";
string collectionname = StringHelper.CalculateMD5Hash(domain).ToLower().Substring(0, 1);
drl.Add(aList[idx]);
while (idx < (aList.Count - 1) && aList[idx].zone == aList[idx + 1].zone)
{
drl.Add(aList[idx + 1]);
idx++;
}
if (drl.Count > 1)
{
foreach (AuthoritiesSimple a in Row2Authorities(drl))
{
ala[Int32.Parse(collectionname, System.Globalization.NumberStyles.HexNumber)].Add(a);
}
}
dl.Clear();
drl.Clear();
domain = "";
idx++;
}
Console.WriteLine("modellist convert;index={1};use time={0};", watch.ElapsedMilliseconds, index);
try
{
var client = DriverConfiguration.Client;
var db = client.GetDatabase(DriverConfiguration.DatabaseNamespace.DatabaseName);
for (int i = 0; i < 16; i++)
{
IMongoCollection <AuthoritiesSimple> collection = db.GetCollection <AuthoritiesSimple>(i.ToString("x"));
if (ala[i].Count > 0)
{
collection.InsertMany(ala[i]);
}
}
Console.WriteLine("mongodb inserted;index={1};use time={0};", watch.ElapsedMilliseconds, index);
}
catch (Exception ex)
{
Console.WriteLine("Insert Data" + ex.Message);
}
index = index + 20001;
Console.WriteLine("min={0};max={1};index={2};use time={3}", min, max, index, watch.ElapsedMilliseconds);
Console.WriteLine("==============================================");
} while (index < max);
Console.WriteLine("end min={0};max={1};index={2};use time {3}", min, max, index, watch.ElapsedMilliseconds);
watch.Stop();//停止计时
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
