private void txtStart_Click(object sender, EventArgs e)
{
obj = new object();
for (int i = 1; i <= 1; i++)
{
System.Threading.ThreadPool.QueueUserWorkItem(FileToDatabase, i);
}
return;
thread = Convert.ToInt32(txtThread.Text);
SqlDB.SqlConnString = txtDataBase.Text;
SqlDB.ExecSql("update baseversion set Finish=0 where Finish=1");
obj = new object();
VersionLock = new object();
redisClient = new ServiceStack.Redis.IRedisClient[thread + 1];
redisClient[thread] = new RedisOperation().Client;
for (int i = 1; i <= thread; i++)
{
System.Threading.ThreadPool.QueueUserWorkItem(FileToDatabase, i);
}
return;
System.Threading.ThreadPool.QueueUserWorkItem(WriteData, "写进程");
System.Threading.ThreadPool.QueueUserWorkItem(WriteRedis, "Redis保存");
}
public Version GetVerion(string nameFormat)
{
lock (VersionLock)
{
Version version = new Version();
DataTable dt = SqlDB.GetDataSetText("select * from baseversion where finish=0").Tables[0];
foreach (DataRow dataRow in dt.Rows)
{
if (SqlDB.ExecSql(string.Format("update baseversion set finish=1 where versionname='{0}' and finish=0", dataRow["versionname"].ToString().Replace("'", "''"))))
{
version.VersionName = dataRow["versionname"].ToString();
version.VersionId = Convert.ToInt32(dataRow["version"]);
version.Row = Convert.ToInt32(dataRow["row"]);
return(version);
}
}
while (true)
{
int max = 0;
DataTable dtMax = SqlDB.GetDataSetText("select Max(version) from baseversion").Tables[0];
if (dtMax.Rows.Count > 0)
{
if (dtMax.Rows[0][0] == DBNull.Value)
{
max = 1;
}
else
{
max = Convert.ToInt32(dtMax.Rows[0][0]) + 1;
}
}
version.VersionName = string.Format(nameFormat, max.ToString("D3"));
if (SqlDB.ExecSql(string.Format("Insert into baseversion(versionname,version,row,finish) values('{0}',{1},{2},{3})", version.VersionName, max, 0, 1)))
{
version.VersionId = max;
version.Row = 0;
version.Finish = 1;
return(version);
}
}
}
}
public void FileToDatabase1(object obj)
{
SqlDB.SqlConnString = txtDataBase.Text;
//int i=26;
Int32 redisIndex = (Int32)obj - 1;
redisClient[redisIndex] = new RedisOperation().Client;
WriterFile(obj.ToString() + "线程开始同步");
Version version = GetVerion("{0}_登录成功(1)_.txt");
string fileFullPath = txtFilePath.Text + "\\" + version.VersionName;
for (; File.Exists(fileFullPath);)
{
fileFullPath = txtFilePath.Text + "\\" + version.VersionName;
WriterFile(obj.ToString() + "线程开始同步" + version.VersionName);
StreamReader sr = new StreamReader(fileFullPath, Encoding.Default);
String line;
int successNumber = 0;
int repeatNumber = 0;
int totaleNumber = 0;
int indexes = 0;
DateTime currentTime = DateTime.Now;
line = sr.ReadToEnd();
sr.Close();
DataTable dataTable = CreateTable("User163", "username", "password", "datatype");;
string[] lineArray = Regex.Split(line, "\r\n");
for (int i = version.Row; i < lineArray.Length; i++)
{
SleepTempThread();
if (successNumber == 0)
{
dataTable = CreateTable("User163", "username", "password", "datatype");
}
MatchCollection collection = Regex.Matches(lineArray[i], "^(.*)----(.*)$");
if (collection.Count > 0)
{
totaleNumber++;
if (!ExistTempKey(redisIndex, collection[0].Groups[1].Value.ToLower(), collection[0].Groups[2].Value))
{
successNumber++;
DataRow dr = dataTable.NewRow();
dr["username"] = collection[0].Groups[1].Value;
dr["password"] = collection[0].Groups[2].Value;
dr["datatype"] = "163";
dataTable.Rows.Add(dr);
}
else
{
repeatNumber++;
}
if (totaleNumber % 10000 == 0)
{
WriterFile(string.Format(obj.ToString() + "线程：共{0}，成功{1}，失败{2}，用时{3},当前时间：{4}",
totaleNumber, successNumber, repeatNumber, DateTime.Now - currentTime, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")));
currentTime = DateTime.Now;
}
if (successNumber >= 10000)
{
WriteDataQueue(dataTable);
SqlDB.ExecSql(string.Format("update baseversion set row={0} where versionname='{1}'", lineArray.Length, version.VersionName));
successNumber = 0;
}
}
}
if (dataTable.Rows.Count > 0)
{
WriteDataQueue(dataTable);
SqlDB.ExecSql(string.Format("update baseversion set row={0} where versionname='{1}'", lineArray.Length, version.VersionName));
}
lineArray = null;
GC.Collect();
SqlDB.ExecSql(string.Format("update baseversion set row={0},Finish=2,endtime=getdate() where versionname='{1}'", totaleNumber, version.VersionName));
version = GetVerion("{0}_登录成功(1)_.txt");
}
WriterFile(obj + "线程结束同步");
}