/*
* /// <summary>
* /// Insertion des elements de la table With dans la table Target
* /// la liste des PK à utiliser est donné en parametre
* /// </summary>
* /// <param name="targetSchemaName"></param>
* /// <param name="targetTableName"></param>
* /// <param name="withSchemaName"></param>
* /// <param name="withTableName"></param>
* /// <param name="excluded_PKs_Table"></param>
* /// <param name="included_PKs_Table"></param>
* /// <param name="date_format"></param>
* /// <param name="connection_timeout"></param>
* /// <returns></returns>
* ///
*/
public override int BulkCopydataCsvFile(string sourceCsvFile, DatabaseTable targetTableName, string excluded_PKs_Table = null, string included_PKs_Table = null, string csvFileEncoding = "UTF8", string date_format = "DMY", int connection_timeout = -1)
{
Encoding enc = csvFileEncoding.GetEncoding();
List <string> columns = new List <string>();
Stream s = new FileStream(sourceCsvFile, FileMode.Open);
using (var reader = new FileDataReader(s, null, enc))
{
try
{
//TODO
Connection.Open();
}
catch (Exception e)
{
throw e;
}
finally
{
Connection.Close();
}
}
return(0);
}
static void Main()
{
var dataReader = new FileDataReader();
var monitoredExtension = new MonitoredDataReaderExtension(dataReader, null);
var authorizedExtension = new AuthorizedDataReaderExtension(dataReader, "key123asd", monitoredExtension);
dataReader.AddExtension(authorizedExtension);
dataReader.RemoveExtension(typeof(MonitoredDataReaderExtension));
string[] result = {};
try
{
result = dataReader.ReadData("data.txt");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
foreach (var line in result)
{
Console.WriteLine(line);
}
Console.Read();
}
private List <int> GenerateTripIdList()
{
var stopTimesPath =
Path.Combine(_baseDirectoryPath, @"Data Files\stop_times.txt"); // files from google transit (GTFS file)
if (!File.Exists(stopTimesPath))
{
Console.WriteLine(this + " Error! File stop_times.txt does not exist!");
return(null);
}
else
{
FileDataReader fdr = new FileDataReader();
var stopTimesData = fdr.ImportData(stopTimesPath, ',', true);
var tripsIdList = new List <int>();
foreach (var singleData in stopTimesData)
{
if (!tripsIdList.Contains(int.Parse(singleData[0])))
{
tripsIdList.Add(
int.Parse(singleData[0])); //adds the trip_id if it doesn't exist yet in trips_id_list
}
}
return(tripsIdList);
}
}
void WriteOutReadIn()
{
string path = "MikeLiuBidirectional.txt";
using (FileDataWriter <string> w = new FileDataWriter <string>(path))
{
w.Append("{First: 'Mike', Last: 'Liu'}");
w.Append("{First: 'Sergey', Last: 'Shandar'}");
Assert.True(File.Exists(path));
}
string sampleText = File.ReadAllText(path);
Assert.Equal("\"{First: 'Mike', Last: 'Liu'}\"\0\"{First: 'Sergey', Last: 'Shandar'}\"\0", sampleText);
//now read using the functionality for FileDatareader
using (FileDataReader <string> r = new FileDataReader <string>(path))
{
(bool, string)currentTuple = r.Read();
Assert.Equal(true, currentTuple.Item1);
Assert.Equal("{First: 'Mike', Last: 'Liu'}", currentTuple.Item2);
currentTuple = r.Read();
Assert.Equal(true, currentTuple.Item1);
Assert.Equal("{First: 'Sergey', Last: 'Shandar'}", currentTuple.Item2);
currentTuple = r.Read();
Assert.Equal(false, currentTuple.Item1);
Assert.Equal(null, currentTuple.Item2);
}
File.Delete(path);
}
public IDataReader GetDataReader(IInputFile inputFile)
{
if (inputFile == null)
{
throw new ArgumentNullException("inputFile");
}
IDataReader dataReader;
if (_dataReaders.TryGetValue(inputFile, out dataReader))
{
return(dataReader);
}
dataReader = new FileDataReader(new DataPacket(inputFile));
// Add buffering decorator
dataReader = new BufferedDataReader(dataReader, BufferSize);
// Add progress reporting decorator
IProgressReporter progressReporter = new FileScannerProgressReporter(_fileScanner, dataReader.Length);
dataReader = new ProgressDataReader(dataReader, progressReporter);
_dataReaders.Add(inputFile, dataReader);
return(dataReader);
}
public void DataShouldReflectTheValues()
{
Stream s = new MemoryStream(1000);
for (int x = 0; x < 1; x++)
{
AddRecordToStream(s, "10,2010-05-05 10:00");
}
s.Position = 0;
FileDataColumn[] cols = new[]
{
new FileDataColumn {
ColumnName = "First", ColumnType = typeof(int)
},
new FileDataColumn {
ColumnName = "Second", ColumnType = typeof(DateTime)
}
};
IDataReader dataReader = new FileDataReader(s, cols, '\n', ',', Encoding.Unicode, null);
bool hasRecords = dataReader.Read();
int first = (int)dataReader[0];
Assert.Equals(first, 10);
DateTime second = (DateTime)dataReader[1];
Assert.Equals(second, new DateTime(2010, 5, 5, 10, 0, 0));
}
public void RecordManipulatorShouldSuccessFullyManipulate()
{
Stream s = new MemoryStream(1000);
for (int x = 0; x < 10; x++)
{
AddRecordToStream(s, string.Format("{0}\n", (x * 10)));
}
s.Position = 0;
FileDataColumn[] cols = new[]
{
new FileDataColumn {
ColumnName = "First", ColumnType = typeof(int)
}
};
IDataReader dataReader = new FileDataReader(
s,
cols,
'\n',
';',
Encoding.Unicode,
record =>
{
int currentValue = record.GetInt32(0);
record.SetValue(0, currentValue * 2);
});
for (int x = 0; x < 10; x++)
{
dataReader.Read();
Assert.Equals(dataReader[0], x * 10 * 2);
}
}
public void ReadShouldReturnTrueIfThereAreAValidRecord()
{
Stream s = new MemoryStream(1000);
for (int x = 0; x < 1; x++)
{
AddRecordToStream(s, "10,2010-05-05 10:00");
}
s.Position = 0;
FileDataColumn[] cols = new[]
{
new FileDataColumn {
ColumnName = "First", ColumnType = typeof(int)
},
new FileDataColumn {
ColumnName = "Second", ColumnType = typeof(DateTime)
}
};
IDataReader dataReader = new FileDataReader(s, cols, '\n', ',', Encoding.Unicode, null);
bool hasRecords = dataReader.Read();
Assert.IsTrue(hasRecords);
}
/// <summary>
/// Event which is raised when the Background Worker is started.
/// </summary>
/// <param name="sender">Sender object for the event.</param>
/// <param name="args">Event args for the event.</param>
private void UploadFile(object sender, DoWorkEventArgs args)
{
string connectionString = _configManager.GetConnectionString();
if (!string.IsNullOrEmpty(connectionString))
{
IFileDataReader fileReader = new FileDataReader();
IFileDataWriter fileWriter = new FileDataWriter(connectionString);
System.Diagnostics.Stopwatch stopWatch = System.Diagnostics.Stopwatch.StartNew();
string fileExtension = System.IO.Path.GetExtension(SelectedFileName);
if (fileExtension == ".csv")
{
foreach (var v in fileReader.ReadDataFromCSV(SelectedFileName))
{
fileWriter.WriteDataToSQL("AccountTransactionData", v.DataTable, GetColumnMappings());
_fileUploaderBackgroundWorker.ReportProgress((int)CalculatePercentage(v.TotalBytes, v.BytesRead), string.Format("{0} Lines Processed, {1} Lines Imported", v.LinesProcessed, v.LinesImported));
SkippedLinesMessage = ProcessSkippedLines(v.SkippedLines);
}
logger.Info(string.Format("Processing of File Completed in {0} seconds", stopWatch.Elapsed.Seconds));
}
else if (fileExtension == ".xlsx")
{
//Call the ReadDataFromExcel method.
}
}
else
{
logger.Error("Connection String not valid in the config file");
_messageBoxService.ShowMessageBox("Please check the Connection string in Configuration file", "AccountTransactionUploadApp");
}
}
public void GetDetailFor1Arg()
{
var reader = new FileDataReader(_fileData);
var result = Assert.Throws <ArgumentException>(() => reader.GetDetail(new string[] { "-v" }));
Assert.NotNull(result);
Assert.Equal($"Two arguments must be provided: the first to define the mode, the second to specify the file.{Environment.NewLine}Arguments recieved: '-v'", result.Message);
}
public void CSVFileReader_DoubleInitialization()
{
FileDataReader reader = new FileDataReader();
string filePath = GetTempFilePath();
reader.Open(filePath);
Assert.Throws <InvalidOperationException>(() => reader.Open(GetTempFilePath()));
reader.Close();
}
public static IReadOnlyCollection <Period> GetHistoryFromFile()
{
string filePath = @"D:\\InterviewTest\InterviewTest.DriverData\DataFiles\HistoryData.json";
IDataFileReader fileDataReader = new FileDataReader();
string historyData = fileDataReader.ReadFileData(filePath);
IDataParser jsondataParser = new JsonDataParser();
return(jsondataParser.ParseData <IReadOnlyCollection <Period> >(historyData));
}
public void GetDetailForValidSizeStrings(string detailTypeString)
{
var reader = new FileDataReader(_fileData);
var result = reader.GetDetail(detailTypeString, "c:/test.txt");
Assert.NotNull(result);
Assert.NotEmpty(result);
Assert.True(int.TryParse(result, out int size)); //the size string can be parsed as an int
}
public void CSVFileReader_MultipleCleanups()
{
FileDataReader reader = new FileDataReader();
string filePath = GetTempFilePath();
reader.Open(filePath);
Assert.DoesNotThrow(() => reader.Close());
Assert.DoesNotThrow(() => reader.Close());
}
public void GetDetailForInvalidStrings_argsArray(string detailTypeString)
{
var reader = new FileDataReader(_fileData);
var result = Assert.Throws <ArgumentException>(() => reader.GetDetail(new string[] { detailTypeString, "c:/test.txt" }));
Assert.NotNull(result);
Assert.Contains(detailTypeString, result.Message);
Assert.Equal($"'{detailTypeString}' is not a valid detail type argument.{Environment.NewLine}" +
$"Please use '-v' or '--version' as the first argument to read the file version or '-s' or '--size' as the first argument to read the file size.", result.Message);
}
static void Main(string[] args)
{
string workingDirectory = Environment.CurrentDirectory;
string projectDirectory = Directory.GetParent(Directory.GetParent(workingDirectory).Parent.FullName).FullName;
string fileName = "Pedidos.csv";
IFileExistValidator fileExistValidator = new FileExistValidator();
IFileDataReader fileDataReader = new FileDataReader();
IClock clock = new Clock();
ProcesadorArchivoPedidos procesadorArchivoPedidos =
new ProcesadorArchivoPedidos(projectDirectory, fileName, fileExistValidator, fileDataReader, clock);
procesadorArchivoPedidos.ProcesarArchivo();
}
public void CSVFileReader_SuccessfulMultipleReads_LessData()
{
string filePath = GetTempFilePath();
WriteData(filePath, "data1");
FileDataReader reader = new FileDataReader();
reader.Open(filePath);
Assert.DoesNotThrow(() => reader.ReadLine());
Assert.DoesNotThrow(() => reader.ReadLine());
reader.Close();
}
public void GetDetailForValidVersionStrings_argsArray(string detailTypeString)
{
var reader = new FileDataReader(_fileData);
var result = reader.GetDetail(new string[] { detailTypeString, "c:/test.txt" });
Assert.NotNull(result);
Assert.NotEmpty(result);
var parts = result.Split('.');
Assert.Equal(3, parts.Length); //there are three parts
Assert.All(parts, p =>
Assert.True(int.TryParse(p, out int versionPart)) //each part can be parsed as an int.
);
}
private List <string[]> GenerateListData(string path)
{
List <string[]> listData = null;
if (!File.Exists(path))
{
Console.WriteLine(this + " Error! File at " + path + " does not exist!");
}
else
{
FileDataReader fdr = new FileDataReader();
listData = fdr.ImportData(path, ',', true);
}
return(listData);
}
public void CSVFileReader_SuccessfulRead()
{
string filePath = GetTempFilePath();
WriteData(filePath, "somedata");
FileDataReader reader = new FileDataReader();
reader.Open(filePath);
string dataRead = null;
Assert.DoesNotThrow(() => dataRead = reader.ReadLine());
Assert.AreEqual("somedata", dataRead);
reader.Close();
}
static void Main(string[] args)
{
//没有命令无法执行
if (args == null || args.Length == 0)
{
Console.WriteLine("No commond !");
return;
}
//args = new string[] { "-x", "-c", "-l" };
//获取指令蕴含信息
CommondInfo info = CommondReader.GetInfo(args);
//错误处理
if (info.isError)
{
ErrorHanding(info.errorType);
return;
}
//生成计数器
WordCounter counter = new WordCounter();
counter.AddFunction(info.funcs);
string data;
//获取信息
if (!FileDataReader.GetDataByPath(info.filePath, out data))
{
Console.WriteLine("Open file failed !");
return;
}
//设置信息并进行计数
counter.SetString(data);
//输出
Console.WriteLine(counter.GetAllLogs());
//Console.ReadKey();
}
public void BoolShouldBeSupported()
{
Stream s = new MemoryStream(1000);
for (int x = 0; x < 1; x++)
{
AddRecordToStream(s, "true");
}
s.Position = 0;
FileDataColumn[] cols = new[]
{
new FileDataColumn {
ColumnName = "First", ColumnType = typeof(bool)
}
};
IDataReader dataReader = new FileDataReader(s, cols, '\n', ';', Encoding.Unicode, null);
bool hasRecords = dataReader.Read();
bool boolValue = (bool)dataReader[0];
Assert.IsTrue(boolValue);
}
public void DateTimeShouldBeSupported()
{
Stream s = new MemoryStream(1000);
for (int x = 0; x < 1; x++)
{
AddRecordToStream(s, "2010-05-05 10:00:01.005");
}
s.Position = 0;
FileDataColumn[] cols = new[]
{
new FileDataColumn {
ColumnName = "First", ColumnType = typeof(DateTime)
}
};
IDataReader dataReader = new FileDataReader(s, cols, '\n', ',', Encoding.Unicode, null);
dataReader.Read();
DateTime dateTime = (DateTime)dataReader[0];
Assert.Equals(dateTime, new DateTime(2010, 5, 5, 10, 0, 1, 5));
}
public void DoubleShouldBeSupported()
{
Stream s = new MemoryStream(1000);
for (int x = 0; x < 1; x++)
{
AddRecordToStream(s, "10,0008\n");
}
s.Position = 0;
FileDataColumn[] cols = new[]
{
new FileDataColumn {
ColumnName = "First", ColumnType = typeof(double)
}
};
IDataReader dataReader = new FileDataReader(s, cols, '\n', ';', Encoding.Unicode, null);
bool hasRecords = dataReader.Read();
double number = (double)dataReader[0];
Assert.Equals(number, 10.0008);
}
private void Load()
{
var watch = Stopwatch.StartNew();
Console.WriteLine(this + "Loading all the necessary data...");
Console.WriteLine(this + "Urban routes only:" + _urbanOnly);
_baseDirectoryPath = Directory
.GetParent(Directory.GetParent(Directory.GetParent(Environment.CurrentDirectory).FullName).FullName)
.FullName;
var stopsPath = Path.Combine(_baseDirectoryPath, @"Data Files\stops.txt"); //files from google transit (GTFS file)
var routesPath = Path.Combine(_baseDirectoryPath, @"Data Files\routes.txt"); //files from google transit (GTFS file)
var demandsPath = Path.Combine(_baseDirectoryPath, @"Data Files\demands.csv");
var stopTimesPath =
Path.Combine(_baseDirectoryPath, @"Data Files\stop_times.txt"); // files from google transit (GTFS file)
string tripsPath = Path.Combine(_baseDirectoryPath, @"Data Files\trips.txt");
var tripStopsPath =
Path.Combine(_baseDirectoryPath, @"Data Files\trip_stops.txt"); //file generated from stop_times.txt and stop.txt
var routesData = GenerateListData(routesPath);
LoadRoutes(routesData);
var tripsData = GenerateListData(tripsPath);
LoadTrips(tripsData);
var stopsData = GenerateListData(stopsPath);
LoadStops(stopsData);
var demandsData = GenerateListData(demandsPath);
var stopTimesDataList = GenerateListData(stopTimesPath);
FileDataExporter dataExporter = new FileDataExporter();
if (!File.Exists(tripStopsPath)
) //if the file doesn't exists, generate the dictionary required to sort the stops in ascending order then export to txt, then reads from the txt the next time the program is executed (to save computational time)
{
var tripsStopTupleDictionary = GenerateTripStopTuplesDictionary(stopTimesDataList);
dataExporter.ExportTripStops(tripsStopTupleDictionary, tripStopsPath);
}
FileDataReader fdr = new FileDataReader();
var tripsStopData = fdr.ImportData(tripStopsPath, ',', true);
LoadStopsIntoTrips(tripsStopData);
LoadTripStartTimes(tripsStopData);
AssignUrbanStops();
//dataExporter.ExportStops(Stops, Path.Combine(Environment.CurrentDirectory, @"stops.txt"));
//dataExporter.ExportTrips(Routes, Path.Combine(Environment.CurrentDirectory, @"trips.txt"));
//dataExporter.ExportTripStopSequence(Routes, Path.Combine(Environment.CurrentDirectory, @"trip_stops.txt"));
//dataExporter.ExportTripStartTimes(Routes, Path.Combine(Environment.CurrentDirectory, @"trip_start_times.txt"));
if (_urbanOnly)
{
Routes = Routes.FindAll(r => r.UrbanRoute); // only urban routes
Trips = Trips.FindAll(t => t.Route.UrbanRoute == true); // only urban trips
Stops = Stops.FindAll(s => s.IsUrban); //only urban stops
}
LoadStopDemands(demandsData);
RemoveDuplicateTrips();
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
Console.WriteLine(this + "All the necessary data was successfully generated in " + elapsedMs * 0.001 + " seconds.");
string str;
str = _urbanOnly ? "Urban " : "";
Console.WriteLine(this + "Total of " + str + "Routes:" + Routes.Count);
Console.WriteLine(this + "Total of " + str + "Route Trips:" + Routes.Sum(r => r.Trips.Count));
Console.WriteLine(this + "Total of " + str + "Stops:" + Stops.Count);
}
public void CSVFileReader_CleanupWithoutInitialization()
{
FileDataReader reader = new FileDataReader();
Assert.DoesNotThrow(() => reader.Close());
}
public MainViewModel()
{
InitCommand = new Command(() =>
{
shapes.Clear();
FileDataReader.ReadDataFile(dataFilePath)
.Select(line => DataParser.ParseDataLine(line))
.ToList()
.ForEach(data =>
{
if (data.Count == 6)
{
Models.Point a = new Models.Point(data[0], data[1]);
Models.Point b = new Models.Point(data[2], data[3]);
Models.Point c = new Models.Point(data[4], data[5]);
shapes.Add(new Triangle(a, b, c));
}
else if (data.Count == 8)
{
Models.Point a = new Models.Point(data[0], data[1]);
Models.Point b = new Models.Point(data[2], data[3]);
Models.Point c = new Models.Point(data[4], data[5]);
Models.Point d = new Models.Point(data[6], data[7]);
shapes.Add(new Quadrilateral(a, b, c, d));
}
});
});
PlainDrawCommand = new Command(() =>
{
OnPropertyChanged(nameof(Lines));
});
ColorDrawCommand = new Command(() =>
{
double largestQuadPerimeter = shapes
.FindAll(shape => shape is Quadrilateral)
.OrderByDescending(quad => quad.Perimeter)
.First().Perimeter;
List <Triangle> halfPerimeterTriangles = shapes
.FindAll(shape => shape is Triangle && shape.Perimeter < largestQuadPerimeter / 2)
.Cast <Triangle>().ToList();
halfPerimeterTriangles.ForEach(triangle => triangle.Color = Colors.Blue);
List <Triangle> twoLargestTriangles = halfPerimeterTriangles
.OrderByDescending(triangle => triangle.Perimeter)
.Take(2).ToList();
twoLargestTriangles.ForEach(triangle => triangle.Color = Colors.Green);
if (twoLargestTriangles.Count == 2)
{
twoLargestIntersect = twoLargestTriangles[0].Intersects(twoLargestTriangles[1]);
}
OnPropertyChanged(nameof(Lines));
});
IntersectCommand = new Command(() =>
{
MessageBox.Show(twoLargestIntersect.ToString());
});
ExitCommand = new Command(() =>
{
Environment.Exit(0);
});
}
public void OnSetup()
{
_fileDataReader = new FileDataReader();
}
public void CSVFileReader_ReadWithoutInitialization()
{
FileDataReader reader = new FileDataReader();
Assert.Throws <InvalidOperationException>(() => reader.ReadLine());
}
