public static void WriteToFile(NpgsqlDataReader reader, string filename)
{
using (var tempFile = File.Create(filename))
{
using (var writer = new StreamWriter(tempFile))
{
var columns = reader.GetColumnSchema();
for (int i = 0; i < columns.Count; i++)
{
if (i == 0)
{
writer.Write(columns[i].ColumnName);
}
else
{
writer.Write("\t{0}", columns[i].ColumnName);
}
}
writer.WriteLine();
while (reader.Read())
{
for (int i = 0; i < columns.Count; i++)
{
Type type = reader.GetFieldType(i);
var method = reader.GetType().GetMethod("GetFieldValue", new Type[] { typeof(int) });
var genericMethod = method.MakeGenericMethod(type);
var value = genericMethod.Invoke(reader, new object[] { i });
string valueHolder = "NULL";
if (value != null)
{
valueHolder = value.ToString();
}
if (i == 0)
{
writer.Write(valueHolder);
}
else
{
writer.Write("\t{0}", valueHolder);
}
}
writer.WriteLine();
}
writer.Flush();
writer.Close();
}
}
}
public async static Task StreamSqlToGCS(NpgsqlDataReader reader, StorageClient storageClient, string filename, string bucketName)
{
using (MemoryStream ms = new MemoryStream())
{
new Thread(() =>
{
Console.WriteLine("Start reading DB.");
var columns = reader.GetColumnSchema();
for (int i = 0; i < columns.Count; i++)
{
if (i == 0)
{
//Console.Write(columns[i].ColumnName);
WriteStringToStream(ms, columns[i].ColumnName);
}
else
{
//Console.Write("\t{0}", columns[i].ColumnName);
WriteStringToStream(ms, string.Format("\t{0}", columns[i].ColumnName));
}
}
//Console.WriteLine();
WriteStringToStream(ms, "\n");
while (reader.Read())
{
for (int i = 0; i < columns.Count; i++)
{
Type type = reader.GetFieldType(i);
var method = reader.GetType().GetMethod("GetFieldValue", new Type[] { typeof(int) });
var genericMethod = method.MakeGenericMethod(type);
var value = genericMethod.Invoke(reader, new object[] { i });
string valueHolder = "NULL";
if (value != null)
{
valueHolder = value.ToString();
}
if (i == 0)
{
//Console.Write(valueHolder);
WriteStringToStream(ms, valueHolder);
}
else
{
//Console.Write("\t{0}", valueHolder);
WriteStringToStream(ms, string.Format("\t{0}", valueHolder));
}
}
//Console.WriteLine();
WriteStringToStream(ms, "\n");
//Thread.Sleep(200);
}
Console.WriteLine("Finished reading DB.");
}).Start();
Thread.Sleep(5000);
await storageClient.UploadObjectAsync(
bucketName,
filename,
"text/html",
ms
);
//Thread.Sleep(5000);
using (var outputFile = File.OpenWrite("out-" + filename))
{
await storageClient.DownloadObjectAsync(bucketName, filename, outputFile);
}
}
Console.WriteLine("File uploaded.");
}
public async static Task StreamToGCS()
{
//Task.Run(async () => { await test(); }).GetAwaiter().GetResult();
string connectionString = "Host=localhost;Port=5701;Database=hiring;Username=tri;Password=1hEWZ4GeN24c";
Connection = new NpgsqlConnection(connectionString);
/**
* Needs Cleanup
* */
NpgsqlCommand cmd = new NpgsqlCommand();
cmd.CommandText = "SELECT * FROM public.language;";
cmd.Connection = Connection;
cmd.Connection.Open();
MemoryStream ms = new MemoryStream();
/**
* Needs Cleanup
* */
NpgsqlDataReader reader = cmd.ExecuteReader();
new Thread(() => {
var columns = reader.GetColumnSchema();
while (reader.Read())
{
for (int i = 0; i < columns.Count; i++)
{
Type type = reader.GetFieldType(i);
var method = reader.GetType().GetMethod("GetFieldValue", new Type[] { typeof(int) });
var genericMethod = method.MakeGenericMethod(type);
var value = genericMethod.Invoke(reader, new object[] { i });
string valueHolder = "NULL";
if (value != null)
{
valueHolder = value.ToString();
}
if (i == 0)
{
Console.Write(valueHolder);
WriteStringToStream(ms, valueHolder);
}
else
{
Console.Write("\t{0}", valueHolder);
WriteStringToStream(ms, string.Format("\t{0}", valueHolder));
}
}
Console.WriteLine();
WriteStringToStream(ms, "\n");
//Thread.Sleep(200);
}
}).Start();
var credentialsPath = "auth\\gd-hiring.json";
var credentialsJson = File.ReadAllText(credentialsPath);
var googleCredential = GoogleCredential.FromJson(credentialsJson);
var storageClient = StorageClient.Create(googleCredential);
storageClient.Service.HttpClient.Timeout = new TimeSpan(1, 0, 0);
var bucketName = "gd-hiring-tri";
await storageClient.UploadObjectAsync(
bucketName,
"public.language_2018-05-11.tsv",
"text/html",
ms
);
Thread.Sleep(5000);
using (var outputFile = File.OpenWrite("temp-output.tsv"))
{
storageClient.DownloadObject(bucketName, "public.language_2018-05-11.tsv", outputFile);
}
Console.WriteLine("Done??");
}
