public static void RunCharDelim(
string DropDirectory,
string DBConnect,
string UserName,
bool toUpper,
int subtractYearLimit,
int addYearLimit,
SqlConnection c,
int ReportLineCountInterval,
bool isCSVPlusQuoteDelim)
{
DirectoryInfo di = new DirectoryInfo(DropDirectory);
string bakDir = di.FullName + "\\BAK\\";
if (!Directory.Exists(bakDir))
{
Directory.CreateDirectory(bakDir);
}
FileInfo[] fins = di.GetFiles("*.*", SearchOption.TopDirectoryOnly);
List <FileInfo> fins2 = (from ff in fins
where (
ff.Extension.ToLower() == ".csv" ||
ff.Extension.ToLower() == ".tab" ||
ff.Extension.ToLower() == ".pipe" ||
ff.Extension.ToLower() == ".hat") &&
ff.LastWriteTime < DateTime.Now.AddSeconds(-0.5)
orderby ff.Length descending
select ff).ToList <FileInfo>();
if (fins2.Count < 1)
{
return;
}
DateTime stm = DateTime.Now;
File.WriteAllText(DropDirectory + "start.time", stm.ToString());
foreach (FileInfo f in fins2)
{
bool isCsv = false;
char delim = ' ';
if (f.Extension.ToLower() == ".csv")
{
isCsv = true;
delim = ',';
}
else if (f.Extension.ToLower() == ".tab")
{
delim = '\t';
}
else if (f.Extension.ToLower() == ".pipe")
{
delim = '|';
}
else if (f.Extension.ToLower() == ".hat")
{
delim = '^';
}
else
{
continue;
}
string fn = f.Name.Trim();
if (fn.Length > 100)
{
fn = fn.Substring(0, 100);
}
Console.WriteLine(f.Name);
long lineCount = 0;
StreamReader sr = new StreamReader(f.FullName);
string fid = Guid.NewGuid().ToString();
string blnFact = "baleen_" + fid + ".fact";
string blnSize = "baleen_" + fid + ".size";
StreamWriter bf = new StreamWriter(di.FullName + "\\" + blnFact);
string cron = DateTime.Now.ToString();
List <long> hdr = null;
string hdrLine = sr.ReadLine();
if (isCSVPlusQuoteDelim)
{
string[] ssplit = CsvQuoteSplit(hdrLine);
hdr = VectoraParser.MakeList(true, ssplit, subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
else
{
hdr = VectoraParser.MakeList(true, hdrLine.Split(delim), subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
bool wroteLineOne = false;
if (hdr != null)
{
if (hdr.Count > 0)
{
WriteRecord(hdr, bf, cron, UserName, fn, c, isCsv, true, isCSVPlusQuoteDelim);
wroteLineOne = true;
}
}
while (!sr.EndOfStream)
{
lineCount++;
if (lineCount % ReportLineCountInterval == 0)
{
Console.WriteLine("Processing File: " + f.Name);
Console.WriteLine("Fact Processing Line No: " + lineCount.ToString());
}
List <long> dat = null;
string datLine = sr.ReadLine();
if (isCSVPlusQuoteDelim)
{
string[] ssplit = CsvQuoteSplit(datLine);
dat = VectoraParser.MakeList(true, ssplit, subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
else
{
dat = VectoraParser.MakeList(true, datLine.Split(delim), subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
if (dat != null)
{
if (dat.Count > 0)
{
if (!wroteLineOne)
{
WriteRecord(dat, bf, cron, UserName, fn, c, isCsv, true, isCSVPlusQuoteDelim);
wroteLineOne = true;
}
else
{
WriteRecord(dat, bf, cron, UserName, fn, c, isCsv, false, isCSVPlusQuoteDelim);
}
}
}
}
bf.Close();
sr.Close();
File.Copy(f.FullName, bakDir + f.Name, true);
File.Delete(f.FullName);
LoadFact(di.FullName + "\\" + blnFact, DBConnect);
File.Delete(di.FullName + "\\" + blnFact);
}
File.WriteAllText(DropDirectory + "end.time", DateTime.Now.ToString());
}
public static void RunCharDelim(
string DropDirectory,
string DBConnect,
string UserName,
bool toUpper,
int subtractYearLimit,
int addYearLimit,
SqlConnection c,
int ReportLineCountInterval,
bool isCSVPlusQuoteDelim)
{
DirectoryInfo di = new DirectoryInfo(DropDirectory);
string bakDir = di.FullName + "\\BAK\\";
if (!Directory.Exists(bakDir))
{
Directory.CreateDirectory(bakDir);
}
FileInfo[] fins = di.GetFiles("*.*", SearchOption.TopDirectoryOnly);
List <FileInfo> fins2 = (from ff in fins
where (
ff.Extension.ToLower() == ".csv" ||
ff.Extension.ToLower() == ".tab" ||
ff.Extension.ToLower() == ".pipe" ||
ff.Extension.ToLower() == ".hat") &&
ff.LastWriteTime < DateTime.Now.AddSeconds(-0.5)
orderby ff.Length descending
select ff).ToList <FileInfo>();
if (fins2.Count < 1)
{
return;
}
DateTime stm = DateTime.Now;
File.WriteAllText(DropDirectory + "start.time", stm.ToString());
foreach (FileInfo f in fins2)
{
bool isCsv = false;
char delim = ' ';
if (f.Extension.ToLower() == ".csv")
{
isCsv = true;
delim = ',';
}
else if (f.Extension.ToLower() == ".tab")
{
delim = '\t';
}
else if (f.Extension.ToLower() == ".pipe")
{
delim = '|';
}
else if (f.Extension.ToLower() == ".hat")
{
delim = '^';
}
else
{
continue;
}
string fn = f.Name.Trim();
if (fn.Length > 100)
{
fn = fn.Substring(0, 100);
}
Console.WriteLine(f.Name);
long lineCount = 0;
StreamReader sr = new StreamReader(f.FullName);
string fid = Guid.NewGuid().ToString();
string blnFact = "baleen_" + fid + ".fact";
string blnSize = "baleen_" + fid + ".size";
StreamWriter bf = new StreamWriter(di.FullName + "\\" + blnFact);
List <long> hdr = null;
string hdrLine = sr.ReadLine();
if (isCSVPlusQuoteDelim)
{
string[] ssplit = CsvQuoteSplit(hdrLine);
hdr = VectoraParser.MakeList(toUpper, ssplit, subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
else
{
hdr = VectoraParser.MakeList(toUpper, hdrLine.Split(delim), subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
long objectTypeKey = GetID(toUpper, fn, subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
long objectTypeFieldNameKey = GetID(toUpper, "object_type", subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
while (!sr.EndOfStream)
{
lineCount++;
if (lineCount % ReportLineCountInterval == 0)
{
Console.WriteLine("Processing File: " + f.Name);
Console.WriteLine("Fact Processing Line No: " + lineCount.ToString());
}
string objectKeyStr = "H-N-S:" + Guid.NewGuid().ToString();
long objectKey = GetID(toUpper, objectKeyStr, subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
List <long> dat = null;
string datLine = sr.ReadLine();
if (isCSVPlusQuoteDelim)
{
string[] ssplit = CsvQuoteSplit(datLine);
dat = VectoraParser.MakeList(toUpper, ssplit, subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
else
{
dat = VectoraParser.MakeList(toUpper, datLine.Split(delim), subtractYearLimit, addYearLimit, c, isCsv, isCSVPlusQuoteDelim);
}
if (dat != null && hdr != null && hdr.Count == dat.Count)
{
string sqlInsert = "INSERT INTO [fact] ([object_key],[field_key],[value_key],[created_on]) VALUES (<<OBJ_KEY>>,<<FIELD_KEY>>,<<VALUE_KEY>>,'<<CREATED>>')";
string sqlWrite = sqlInsert
.Replace("<<OBJ_KEY>>", objectKey.ToString())
.Replace("<<FIELD_KEY>>", objectTypeFieldNameKey.ToString())
.Replace("<<VALUE_KEY>>", objectTypeKey.ToString())
.Replace("<<CREATED>>", DateTime.Now.ToString());
bf.WriteLine(sqlWrite);
for (int i = 0; i < hdr.Count; i++)
{
long h = hdr[i];
long d = dat[i];
if (d == -1)
{
continue;
}
sqlWrite = sqlInsert
.Replace("<<OBJ_KEY>>", objectKey.ToString())
.Replace("<<FIELD_KEY>>", h.ToString())
.Replace("<<VALUE_KEY>>", d.ToString())
.Replace("<<CREATED>>", DateTime.Now.ToString());
bf.WriteLine(sqlWrite);
}
}
}
bf.Close();
sr.Close();
File.Copy(f.FullName, bakDir + f.Name, true);
File.Delete(f.FullName);
LoadFact(di.FullName + "\\" + blnFact, DBConnect);
File.Delete(di.FullName + "\\" + blnFact);
}
File.WriteAllText(DropDirectory + "end.time", DateTime.Now.ToString());
}