private static void TestReadFile()
{
//create a simple DBF file and output to args[0]
var odbf = new DbfFile(Encoding.GetEncoding(1252));
odbf.Open(Path.Combine(TestPath,"TestNew2.dbf"), FileMode.Open); //State_vars1.dbf county_vars1.dbf
//if (File.Exists("P:\\Development\\Library\\CS\\DbfLib\\DBFSamples\\filesource.txt"))
// File.Delete("P:\\Development\\Library\\CS\\DbfLib\\DBFSamples\\filesource.txt");
var ofs = new FileStream(Path.Combine(TestPath,"filesource.txt"), FileMode.Create);
var osw = new StreamWriter(ofs, Encoding.Default);
//read and print records to screen...
var orec = new DbfRecord(odbf.Header);
for (int i = 0; i < odbf.Header.RecordCount; i++)
{
if (!odbf.Read(i, orec))
break;
osw.WriteLine("index: " + orec.RecordIndex + ": " + orec);
}
/*
while (odbf.ReadNext(orec))
{
osw.WriteLine("index: " + orec.RecordIndex + ": " + orec.ToString());
}
*/
osw.Flush();
osw.Close();
}
public DbfWriter(SocialExplorer.IO.FastDBF.DbfFile dbf, DataFormatSettings dataFormat)
{
_dbf = dbf;
_orec = new DbfRecord(_dbf.Header);
_dataFormat = dataFormat;
_formatter = new CdlValueFormatter(_dataFormat ?? new DataFormatSettings());
}
private static void TestReadHttpConn()
{
//Open a web file...
//-------------------------------------
var oWebDBF = new DbfFile(Encoding.GetEncoding(1252));
var oWebFile = new WebClient();
oWebDBF.Open(oWebFile.OpenRead("http://private.socialexplorer.com/State_vars1.dbf"));
//read and print records to screen...
var orecWeb = new DbfRecord(oWebDBF.Header);
var ofs2 = new FileStream(Path.Combine(TestPath,"Webfile.txt"), FileMode.Create);
var osw2 = new StreamWriter(ofs2);
bool bIsForwardOnly = oWebDBF.IsForwardOnly;
bool bIsReadOnly = oWebDBF.IsReadOnly;
while (oWebDBF.ReadNext(orecWeb))
osw2.WriteLine("index: " + orecWeb.RecordIndex + ": " + orecWeb);
osw2.Flush();
osw2.Close();
oWebDBF.Close();
}
private SocialExplorer.IO.FastDBF.DbfFile OpenDbfRead(IShellContext context)
{
var dbf = new SocialExplorer.IO.FastDBF.DbfFile(Encoding);
var name = context.ResolveFile(context.Replace(Name), ResolveFileMode.Input);
dbf.Open(name, System.IO.FileMode.Open);
return dbf;
}
public static IEnumerable<AddressPartIndexWrite> ParseItems(Stream ms)
{
DbfFile dbfFile = new DbfFile(Encoding.GetEncoding(866));
dbfFile.Open(ms);
try
{
DbfHeader header = dbfFile.Header;
dynamic headerObj = new
{
NAME = FindColumnOrThrow(header, "NAME", maxLen: 40),
SOCR = FindColumnOrThrow(header, "SOCR", maxLen: 10),
CODE = FindColumnOrThrow(header, "CODE", maxLen: 17),
INDEX = FindColumnOrThrow(header, "INDEX", maxLen: 6),
// GNINMB = FindColumnOrThrow(header, "GNINMB", maxLen: 4),
// UNO = FindColumnOrThrow(header, "UNO", maxLen: 4),
// OCATD = FindColumnOrThrow(header, "OCATD", maxLen: 11),
};
IEnumerable<DbfRecord> records = EnumRecords(dbfFile, header);
foreach (var record in records)
{
AddressPartIndexWrite d = EnumRecord(headerObj, record);
if (d != null) yield return d;
}
}
finally
{
dbfFile.Close();
}
}
public static IEnumerable<AddressPartReduction> ParseItems(Stream ms)
{
DbfFile dbfFile = new DbfFile(Encoding.GetEncoding(866));
dbfFile.Open(ms);
try
{
DbfHeader header = dbfFile.Header;
dynamic headerObj = new
{
LEVEL = FindColumnOrThrow(header, "LEVEL", maxLen: 5),
SCNAME = FindColumnOrThrow(header, "SCNAME", maxLen: 10),
SOCRNAME = FindColumnOrThrow(header, "SOCRNAME", maxLen: 29),
// KOD_T_ST = FindColumnOrThrow(header, "KOD_T_ST", maxLen: 3),
};
IEnumerable<DbfRecord> records = EnumRecords(dbfFile, header);
foreach (var record in records)
{
var item = new AddressPartReduction()
{
Level = int.Parse(record[headerObj.LEVEL].Trim()),
Short = record[headerObj.SCNAME].Trim(),
Name = record[headerObj.SOCRNAME].Trim(),
// KOD_T_ST = record[headerObj.KOD_T_ST].Trim(),
};
yield return item;
}
}
finally
{
dbfFile.Close();
}
}
public DbfReader(TableInfo structure, SocialExplorer.IO.FastDBF.DbfFile dbf)
: base(structure)
{
_dbf = dbf;
_array = new string[structure.ColumnCount];
_irec = new DbfRecord(_dbf.Header);
}
protected static IEnumerable<DbfRecord> EnumRecords(DbfFile file, DbfHeader header)
{
var record = new DbfRecord(header);
while (file.ReadNext(record))
{
yield return record;
}
}
static void Main(string[] args)
{
if(args.Length < 2)
{
//print help
Console.WriteLine("\n\n");
Console.WriteLine("Welcome to Social Explorer DBF 2 CSV Utility");
Console.WriteLine("-------------------------------------------------");
Console.WriteLine("\nParameters:");
Console.WriteLine("1. input DBF file");
Console.WriteLine("2. output CSV file");
Console.WriteLine("\nOptional switches:");
Console.WriteLine("/F - format numbers so 5.5000 comes out as 5.5");
Console.WriteLine("/P - padded output, fixed width (/P trumps /F)");
Console.WriteLine("/Q - only output quotes when comma appears in data");
Console.WriteLine("\n\nExample: dbf2csv \"in.dbf\" \"out.csv\" /P /Q");
}
else
{
//check if input DBF file exists...
if(!File.Exists(args[0]))
{
Console.WriteLine("Input file '" + args[0] + "' does not exist!");
return;
}
//create output csv file overwrite if already exists.
if(File.Exists(args[1]))
{
//ask to overwrite:
Console.WriteLine("Output CSV file '" + args[1] + "' already exists.");
Console.WriteLine("Would you like to overwrite it? Press 'Y' for yes: ");
if(Console.ReadKey().KeyChar.ToString().ToUpper() != "Y")
return;
}
bool bSwitchF = false;
bool bSwitchP = false;
bool bSwitchQ = false;
for(int i=0;i<args.Length;i++)
if(args[i] == "/F")
bSwitchF = true;
for (int i = 0; i < args.Length; i++)
if (args[i] == "/P")
bSwitchP = true;
for (int i = 0; i < args.Length; i++)
if (args[i] == "/Q")
bSwitchQ = true;
//open DBF file and create CSV output file...
StreamWriter swcsv = new StreamWriter(args[1], false, Encoding.Default);
DbfFile dbf = new DbfFile(Encoding.UTF8);
dbf.Open(args[0], FileMode.Open);
//output column names
for (int i = 0; i < dbf.Header.ColumnCount; i++)
{
if(dbf.Header[i].ColumnType != DbfColumn.DbfColumnType.Binary &&
dbf.Header[i].ColumnType != DbfColumn.DbfColumnType.Memo)
swcsv.Write((i == 0 ? "": ",") + dbf.Header[i].Name);
else
Console.WriteLine("WARNING: Excluding Binary/Memo field '" + dbf.Header[i].Name + "'");
}
swcsv.WriteLine();
//output values for all but binary and memo...
DbfRecord orec = new DbfRecord(dbf.Header);
while(dbf.ReadNext(orec))
{
//output column values...
if (!orec.IsDeleted)
{
for (int i = 0; i < orec.ColumnCount; i++)
{
if(orec.Column(i).ColumnType == DbfColumn.DbfColumnType.Character)
{
//string values: trim, enclose in quotes and escape quotes with double quotes
string sval = orec[i];
if(!bSwitchP)
sval = orec[i].Trim();
if(!bSwitchQ || sval.IndexOf('"') > -1)
sval = ("\"" + sval.Replace("\"", "\"\"") + "\"");
swcsv.Write(sval);
}
else if(orec.Column(i).ColumnType == DbfColumn.DbfColumnType.Date)
swcsv.Write(orec.GetDateValue(i).ToString("MM-dd-yyyy"));
else
{
if (bSwitchP)
swcsv.Write(orec[i]);
else if(bSwitchF)
swcsv.Write(FormatNumber(orec[i].Trim()));
else
swcsv.Write(orec[i].Trim());
}
//end record with a linefeed or end column with a comma.
if(i < orec.ColumnCount-1)
swcsv.Write(",");
}
//write line...
swcsv.WriteLine();
}
}
//close files...
swcsv.Flush();
swcsv.Close();
dbf.Close();
}
}
/// <summary>
/// Write data in given time range as ArcSWAT dbf file
/// </summary>
/// <param name="startYear"></param>
/// <param name="endYear"></param>
/// <param name="destinationFolder"></param>
/// <returns></returns>
private bool save2ArcSWATdbf(int startYear, int endYear, string destinationFolder)
{
string timeAffix = getTimeAffix();
string pFile = string.Format("{0}\\P{1}{2}.dbf", Path.GetFullPath(destinationFolder), _id, timeAffix); //precipitation
string tFile = string.Format("{0}\\T{1}{2}.dbf", Path.GetFullPath(destinationFolder), _id, timeAffix); //temperature
this.setProgress(0,string.Format("Processing station {0}", _id));
this.setProgress(0, pFile);
this.setProgress(0, tFile);
//create the dbf structure based on ArcSWAT document
DbfFile pDBF = new DbfFile();
pDBF.Open(pFile, FileMode.Create);
pDBF.Header.AddColumn(new DbfColumn("DATE", DbfColumn.DbfColumnType.Date));
pDBF.Header.AddColumn(new DbfColumn("PCP", DbfColumn.DbfColumnType.Number, 5, 1));
DbfFile tDBF = new DbfFile();
tDBF.Open(tFile, FileMode.Create);
tDBF.Header.AddColumn(new DbfColumn("DATE", DbfColumn.DbfColumnType.Date));
tDBF.Header.AddColumn(new DbfColumn("MAX", DbfColumn.DbfColumnType.Number, 5, 1));
tDBF.Header.AddColumn(new DbfColumn("MIN", DbfColumn.DbfColumnType.Number, 5, 1));
DbfRecord pRec = new DbfRecord(pDBF.Header);
DbfRecord tRec = new DbfRecord(tDBF.Header);
int processPercent = 0;
bool hasResults = false;
clearFailureYears();
clearUncompletedYears();
for (int i = startYear; i <= endYear; i++)
{
setProgress(processPercent, string.Format("Downloading data for station: {0}, year: {1}", _id, i));
string resultsForOneYear = this.retrieveAnnualDailyClimateData(i, true);
if (resultsForOneYear.Length == 0)
{
addFailureYear(i);
continue;
}
processPercent += 1;
setProgress(processPercent, "Writing data");
using (CachedCsvReader csv = new CachedCsvReader(new StringReader(resultsForOneYear), true))
{
if (csv.FieldCount >= 27)
{
hasResults = true;
string date = "";
while (csv.ReadNextRecord())
{
date = csv[0];
double p = ClimateString2Double(csv[TOTAL_PRECIPITATION_COL_INDEX]);
pRec[0] = date;
pRec[1] = p.ToString();
pDBF.Write(pRec, true);
double t_max = ClimateString2Double(csv[MAX_T_COL_INDEX]);
double t_min = ClimateString2Double(csv[MIN_T_COL_INDEX]);
tRec[0] = date;
tRec[1] = t_max.ToString();
tRec[2] = t_min.ToString();
tDBF.Write(tRec, true);
}
checkLastDayofYear(date);
}
}
processPercent += 1;
}
pDBF.Close();
tDBF.Close();
return hasResults;
}
private static void TestWriteNewDbf()
{
//create a simple DBF file and output to args[0]
var odbf = new DbfFile(Encoding.GetEncoding(1252));
odbf.Open(Path.Combine(TestPath,"TestNew2.dbf"), FileMode.Create);
//create a header
odbf.Header.AddColumn(new DbfColumn("StrCol", DbfColumn.DbfColumnType.Character, 20, 0));
odbf.Header.AddColumn(new DbfColumn("DecCol1", DbfColumn.DbfColumnType.Number, 5, 1));
odbf.Header.AddColumn(new DbfColumn("DecCol2", DbfColumn.DbfColumnType.Number, 5, 2));
odbf.Header.AddColumn(new DbfColumn("DecCol3", DbfColumn.DbfColumnType.Number, 5, 3));
odbf.Header.AddColumn(new DbfColumn("DecCol4", DbfColumn.DbfColumnType.Number, 15, 5));
odbf.Header.AddColumn(new DbfColumn("NumCol1", DbfColumn.DbfColumnType.Number, 5, 0));
odbf.Header.AddColumn(new DbfColumn("NumCol2", DbfColumn.DbfColumnType.Number, 10, 0));
odbf.Header.AddColumn(new DbfColumn("DateCol1", DbfColumn.DbfColumnType.Date));
odbf.Header.AddColumn(new DbfColumn("BoolCol1", DbfColumn.DbfColumnType.Boolean));
//add some records...
var orec = new DbfRecord(odbf.Header) {AllowDecimalTruncate = true};
orec[0] = "Ahmed Test";
orec[1] = "123.5";
orec[2] = "12.35";
orec[3] = "1.235";
orec[4] = "1235.123456";
orec[5] = "1235";
orec[6] = "123567890";
orec[7] = "11/07/2007";
orec[8] = "f";
odbf.Write(orec, true);
orec[0] = "Stéfanié Singer";
orec[1] = "-1.5";
orec[2] = "-1.35";
orec[3] = "1.235";
orec[4] = "-1235.123";
orec[5] = "15";
orec[6] = "12345"; //put a decimal in integer, we won't throw an exception beacuse we do not test for that.
orec[7] = "2008-12-21";
orec[8] = "f";
odbf.Write(orec, true);
orec[0] = "Stéfanié Singer longer than fits in the DBF record!";
orec[1] = "0.1";
orec[2] = ".12";
orec[3] = ".1";
orec[4] = "";
orec[5] = "-15";
orec[6] = "-12345"; //put a decimal in integer, we won't throw an exception beacuse we do not test for that.
orec[7] = "";
orec[8] = "no";
odbf.Write(orec);
//overwrite first record with last record's data...
orec.RecordIndex = 0;
odbf.Write(orec);
//odbf.Header.RecordCount = 50;
odbf.WriteHeader();
odbf.Close();
//open the same DBF file we just output, and append a few records to it...
odbf.Open(Path.Combine(TestPath,"TestNew2.dbf"), FileMode.Open);
orec.Clear();
orec[0] = "New record added!";
orec[6] = "100";
orec[8] = "t";
odbf.Write(orec, true);
orec[0] = "New record 2";
orec[6] = "104";
orec[8] = "y";
odbf.Write(orec, true);
orec[0] = "New record 3";
orec[6] = "104";
orec[8] = "TRUE";
odbf.Write(orec, true);
if (odbf.Read(0, orec))
{
orec[0] = "modified first record";
odbf.Write(orec, true);
}
//read 3rd record and output to console...
Console.WriteLine(odbf.Read(2).ToString());
//now add a new record, forcing seek to end of file...
orec.Clear();
orec[0] = "New record 4";
orec[6] = "500";
orec[8] = "FALSE";
odbf.Write(orec, true);
odbf.Close();
Console.ReadKey();
}
/// <summary>
/// Initializes a new instance of the <see cref="DbfReader"/> class.
/// </summary>
public DbfReader()
{
this.dbfFile = new DbfFile();
}
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat)
{
string file = context.ResolveFile(context.Replace(Name), ResolveFileMode.Output);
context.OutputMessage("Writing file " + Path.GetFullPath(file));
var dbf = new SocialExplorer.IO.FastDBF.DbfFile(Encoding);
if (File.Exists(file)) File.Delete(file);
dbf.Create(file);
foreach (var col in rowFormat.Columns)
{
DbfColumn.DbfColumnType type;
int len = 0, scale = 0;
switch (col.CommonType.Code)
{
case DbTypeCode.Array:
case DbTypeCode.Generic:
case DbTypeCode.Text:
case DbTypeCode.Xml:
type = DbfColumn.DbfColumnType.Memo;
break;
case DbTypeCode.Blob:
type = DbfColumn.DbfColumnType.Binary;
break;
case DbTypeCode.Datetime:
var dtype = (DbTypeDatetime) col.CommonType;
if (dtype.SubType == DbDatetimeSubType.Date)
{
type = DbfColumn.DbfColumnType.Date;
}
else
{
type = DbfColumn.DbfColumnType.Character;
len = DateTime.UtcNow.ToString("s").Length;
}
break;
case DbTypeCode.Float:
type = DbfColumn.DbfColumnType.Number;
len = 18;
scale = DefaultNumericScale;
break;
case DbTypeCode.Int:
if (AllowFoxProInteger)
{
type = DbfColumn.DbfColumnType.Integer;
}
else
{
type = DbfColumn.DbfColumnType.Number;
len = 18;
}
break;
case DbTypeCode.Logical:
type = DbfColumn.DbfColumnType.Boolean;
break;
case DbTypeCode.Numeric:
type = DbfColumn.DbfColumnType.Number;
len = 18;
scale = ((DbTypeNumeric) col.CommonType).Scale;
break;
case DbTypeCode.String:
var stype = (DbTypeString) col.CommonType;
if (stype.IsBinary)
{
type = DbfColumn.DbfColumnType.Binary;
}
else if (stype.Length <= 254)
{
type = DbfColumn.DbfColumnType.Character;
len = stype.Length;
if (len <= 0) len = DefaultStringLength;
}
else
{
type = DbfColumn.DbfColumnType.Memo;
}
break;
default:
type = DbfColumn.DbfColumnType.Character;
len = DefaultStringLength;
break;
}
dbf.Header.AddColumn(col.Name, type, len, scale);
}
return new DbfWriter(dbf, DataFormat);
}
private DbfFile createDBFGageLocationFile(string file)
{
DbfFile dbf = new DbfFile();
dbf.Open(file, FileMode.Create);
dbf.Header.AddColumn(new DbfColumn("ID", DbfColumn.DbfColumnType.Number, 8, 0)); //integer is a binary Integer type
dbf.Header.AddColumn(new DbfColumn("NAME", DbfColumn.DbfColumnType.Character,8,0));
dbf.Header.AddColumn(new DbfColumn("LAT", DbfColumn.DbfColumnType.Number,8,3));
dbf.Header.AddColumn(new DbfColumn("LONG", DbfColumn.DbfColumnType.Number,8,3));
dbf.Header.AddColumn(new DbfColumn("ELEVATION", DbfColumn.DbfColumnType.Number, 4, 0));//integer is a binary Integer type
return dbf;
}
