//--- Class Methods ---
public static void Main(string[] args)
{
var console = Console.Error;
console.WriteLine("MindTouch CSV2CB - Import CSV files into Couchbase");
console.WriteLine();
// parse command line options
string password = null;
XUri host = null;
string doctype = null;
bool help = false;
string select = null;
var options = new OptionSet {
{ "host=", "Couchbase hostname with bucket name.\nExample: http://example.com:8091/bucket", v => host = XUri.TryParse(v) },
{ "password="******"(optional) Password for Couchbase bucket.", v => password = v },
{ "doctype=", "(optional) Document type identifier (doctype) to add to each converted JSON document.", v => doctype = v },
{ "select=", "(optional) List of CSV columns to import. By default, all columns are imported.", v => select = v },
{ "h|?|help", "This help text", v => help = (v != null) }
};
var filenames = options.Parse(args);
// show help if requested
if (help)
{
WriteOptions(Console.Out, options);
return;
}
// validate arguments
if (host == null)
{
console.WriteLine("ERROR: missing couchbase hostname");
console.WriteLine();
WriteOptions(Console.Out, options);
return;
}
HashSet <string> columns = null;
if (select != null)
{
try {
CsvTable schema = null;
using (var reader = new StringReader(select)) {
schema = CsvTable.NewFromStream(reader);
}
columns = new HashSet <string>();
foreach (var column in schema.Headers)
{
columns.Add(column);
}
} catch (Exception e) {
console.WriteLine("ERROR: {0}", e.Message);
return;
}
}
// create client and verify connection
try {
_client = CreateClient(host, password);
var key = "test:" + StringUtil.CreateAlphaNumericKey(16);
if (!_client.Store(StoreMode.Set, key, "succcess!"))
{
throw new ApplicationException("unable to connect to couchbase server");
}
_client.Remove(key);
} catch (Exception e) {
console.WriteLine("ERROR: {0}", e.Message);
return;
}
// process all files
var loadStopwatch = new Stopwatch();
var convertStopwatch = new Stopwatch();
var importStopwatch = new Stopwatch();
var documentTotal = 0;
var importFailedTotal = 0;
try {
foreach (var filename in filenames)
{
if (!File.Exists(filename))
{
console.WriteLine("ERROR: unable to find '{0}'", filename);
return;
}
try {
// load CSV file
console.Write("Loading file '{0}'...", filename);
loadStopwatch.Start();
var table = CsvTable.NewFromPath(filename, Encoding.UTF8);
loadStopwatch.Stop();
console.WriteLine("done ({0:#,##0} rows)", table.RowCount);
// converting from CSV to JSON documents
console.Write("Converting rows...");
var documents = new List <KeyValuePair <string, string> >();
convertStopwatch.Start();
foreach (var row in table)
{
var value = RowToJson(columns, row, doctype);
if (value == null)
{
continue;
}
var key = StringUtil.CreateAlphaNumericKey(16);
if (!string.IsNullOrEmpty(doctype))
{
key = doctype + ":" + key;
}
documents.Add(new KeyValuePair <string, string>(key, value));
}
convertStopwatch.Stop();
documentTotal += documents.Count;
console.WriteLine("done ({0:#,##0} documents)", documents.Count);
// importing JSON documents
if (documents.Any())
{
console.Write("Importing documents...");
int importFailed;
importStopwatch.Start();
Send(documents, out importFailed);
importStopwatch.Stop();
importFailedTotal += importFailed;
console.WriteLine("done ({0:#,##0} documents)", documents.Count - importFailed);
}
} catch (Exception e) {
console.WriteLine("ERROR: error loading file '{0}': {1}", filename, e.Message);
return;
}
}
} finally {
loadStopwatch.Stop();
convertStopwatch.Stop();
importStopwatch.Stop();
var loadTime = loadStopwatch.Elapsed.TotalSeconds;
var convertTime = convertStopwatch.Elapsed.TotalSeconds;
var importTime = importStopwatch.Elapsed.TotalSeconds;
console.WriteLine();
console.WriteLine("{0:#,##0.000} seconds elapsed (loading: {1:#,##0.000}s, converting: {2:#,##0.000}s, importing: {3:#,##0.000}s)", loadTime + convertTime + importTime, loadTime, convertTime, importTime);
console.WriteLine("{0:#,##0} records loaded ({1:#,##0.000} records/second)", documentTotal, documentTotal / loadTime);
console.WriteLine("{0:#,##0} records converted ({1:#,##0.000} records/second)", documentTotal, documentTotal / convertTime);
console.WriteLine("{0:#,##0} records imported ({1:#,##0.000} records/second)", documentTotal - importFailedTotal, (documentTotal - importFailedTotal) / importTime);
if (importFailedTotal > 0)
{
console.WriteLine("WARNING: {0:#,##0} records failed to import!", importFailedTotal);
}
}
}