public ICollection <T> ParseItems <T>(string header, IEnumerable <string> text) where T : class
{
var joined = string.Join(Environment.NewLine, new[] { header }.Union(text));
using (var stream = new StringReader(joined))
{
var reader = new CsvHelper.CsvFactory().CreateReader(stream);
reader.Configuration.Delimiter = ";";
return(reader.GetRecords <T>().ToList());
}
}
internal static string ToCsv(ICollection <IDictionary <string, string> > data, string delimiter = ",")
{
var factory = new CsvHelper.CsvFactory();
using (StringWriter sw = new StringWriter())
using (var writer = factory.CreateWriter(sw))
{
writer.Configuration.Delimiter = delimiter;
foreach (var kvp in data.First())
{
writer.WriteField(typeof(string), kvp.Key);
}
foreach (var d in data)
{
writer.NextRecord();
foreach (var kvp in d)
{
writer.WriteField(typeof(string), kvp.Value);
}
}
writer.NextRecord();
return(sw.ToString());
}
/* var sb = new StringBuilder();
* bool firstheader = true;
* foreach (var kvp in data.First())
* {
* sb.AppendFormat(firstheader ? @"""{0}""" : @",""{0}""", kvp.Key);
* firstheader = false;
* }
* foreach (var d in data)
* {
* sb.AppendLine();
* bool first = true;
* foreach (var kvp in d)
* {
* sb.AppendFormat(first ? @"""{0}""" : @",""{0}""", kvp.Value);
* first = false;
* }
* }
* return sb.ToString();*/
}
static void Main(string[] args)
{
// Load of type rubbish
var movieProperties = typeof(Movie).GetProperties(BindingFlags.Public | BindingFlags.Instance);
var inserted = 0;
var tStr = typeof(string);
var tInt = typeof(int?);
var tLong = typeof(long?);
var tDouble = typeof(double?);
// Open a connection to MongoDb
var mongoClient = new MongoClient("mongodb://localhost");
var database = mongoClient.GetDatabase("Imdb");
var collection = database.GetCollection <Movie>("Movies");
// Empty the collection
collection.DeleteMany(FilterDefinition <Movie> .Empty);
// Open the CSV
using (var stream = new FileStream("movie_metadata.csv", FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var reader = new StreamReader(stream, Encoding.UTF8))
{
var csvFactory = new CsvHelper.CsvFactory();
var csv = csvFactory.CreateReader(reader, new CsvConfiguration {
Encoding = Encoding.UTF8, HasHeaderRecord = true, TrimFields = true, TrimHeaders = true
});
csv.ReadHeader();
while (csv.Read())
{
var fields = csv.CurrentRecord;
var movie = new Movie();
for (var f = 0; f < fields.Length; f++)
{
var prop = movieProperties[f];
if (prop.PropertyType == tStr)
{
prop.SetValue(movie, fields[f].Trim());
}
if (prop.PropertyType == tInt)
{
prop.SetValue(movie, NullableHelper.ParseNullable <int>(fields[f].Trim(), int.TryParse));
}
if (prop.PropertyType == tLong)
{
prop.SetValue(movie, NullableHelper.ParseNullable <long>(fields[f].Trim(), long.TryParse));
}
if (prop.PropertyType == tDouble)
{
prop.SetValue(movie, NullableHelper.ParseNullable <double>(fields[f].Trim(), double.TryParse));
}
}
// Insert into Mongo
collection.InsertOne(movie);
Console.WriteLine(movie.MovieTitle);
inserted++;
}
}
Console.WriteLine("Done, {0:N0} movies inserted.", inserted);
}
static void RunBenchmark(TextWriter resultWriter, TextWriter logWriter)
{
var fac = new CsvHelper.CsvFactory();
using (var csvWriter = fac.CreateWriter(resultWriter))
{
csvWriter.WriteHeader <BenchmarkResult>();
int[] sizes = { 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000 };
foreach (int sz in sizes)
{
var row = new BenchmarkResult();
var termsCount = sz; row.NumberOfTerms = termsCount;
var varsCount = sz; row.NumberOfVars = varsCount;
var grad = new double[sz];
logWriter.WriteLine("Benchmark for {0} terms and {1} variables", termsCount, varsCount);
logWriter.Write("\tConstructing coefficients ...");
var coefficients = GenerateCoefficients(termsCount, varsCount);
logWriter.WriteLine(" done");
// generate variables
var vars = new Variable[varsCount];
for (var j = 0; j < sz; ++j)
{
vars[j] = new Variable();
}
logWriter.Write("\tGenerating input data ...");
var inputData = new double[1000][];
for (var j = 0; j < inputData.Length; ++j)
{
inputData[j] = RandomDoubles(varsCount);
}
logWriter.WriteLine(" done");
GC.Collect(GC.MaxGeneration, GCCollectionMode.Default, true);
ICompiledTerm compiledTerm = null;
row.CompileMilliseconds = MeasureMsec("Constructing compiled term", logWriter,
() => compiledTerm = ConstructTerm(coefficients, vars));
row.MillisecondsPerManualEval = MeasureMsecPerOp("manual evaluation",
logWriter, inputData.Length, () => inputData.Sum(array => NativeEvaluate(coefficients, array)));
row.MillisecondsPerCompiledEval = MeasureMsecPerOp("AutoDiff compiled evaluation",
logWriter, inputData.Length, () => inputData.Sum(array => compiledTerm.Evaluate(array)));
row.MillisecondsPerCompiledDiff = MeasureMsecPerOp("compiled differentiation",
logWriter, inputData.Length, () =>
{
var sum = 0.0;
foreach (var array in inputData)
{
var val = compiledTerm.Differentiate(array, grad);
sum += val + grad.Sum();
}
return(sum);
});
csvWriter.WriteRecord(row);
}
}
}