public override IEnumerable <List <JSONDocument> > Read(EXIMDataType dataType, string path)
{
List <JSONDocument> items = new List <JSONDocument>();
RecoveryOperationStatus state = base.ValidatePath(path, RecoveryJobType.Import);
if (state.Status == RecoveryStatus.Success)
{
if (ValidateExtension(path))
{
using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read))
using (StreamReader reader = new StreamReader(stream))
using (JsonReader jsonReader = new JsonTextReader(reader))
{
Newtonsoft.Json.JsonSerializer serializer = new Newtonsoft.Json.JsonSerializer();
var itemList = serializer.Deserialize <JSONDocument[]>(jsonReader);
if (itemList != null)
{
foreach (var item in itemList)
{
if (items.Count <= base.ChunkSize)
{
items.Add(item);
}
else
{
items.Add(item);
yield return(items);
items.Clear();
}
}
// if data is less than chunk size
if (items.Count > 0)
{
yield return(items);
}
}
stream.Close();
}
}
else
{
throw new ArgumentException("Invalid file extension");
}
}
else
{
throw new ArgumentException("Invalid file path provided");
}
}
public override RecoveryOperationStatus Write(EXIMDataType dataType, string path, string collection, string fileName, string database, List <IJSONDocument> docList)
{
RecoveryOperationStatus state = base.ValidatePath(path, RecoveryJobType.Export);
if (state.Status == RecoveryStatus.Success)
{
try
{
string file = string.Empty;
if (!string.IsNullOrEmpty(fileName))
{
file = Path.Combine(path, fileName + _fileExtension);
}
else
{
string defaultName = database + "_" + collection;
file = Path.Combine(path, defaultName + _fileExtension);
}
using (StreamWriter sw = new StreamWriter(file))
using (JsonWriter jw = new JsonTextWriter(sw))
{
jw.Formatting = Formatting.Indented;
Newtonsoft.Json.JsonSerializer serializer = new Newtonsoft.Json.JsonSerializer();
serializer.Serialize(jw, docList);
jw.Close();
}
}
catch (Exception exp)
{
if (LoggerManager.Instance.EXIMLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.EXIMLogger.Error("JSONEXIMUtil.Export()", exp.ToString());
}
state.Status = RecoveryStatus.Failure;
state.Message = exp.ToString();
}
return(state);
}
else
{
return(state);
}
}
protected override void BeginProcessing()
{
SessionState s1 = SessionState;
string[] pathChunks = ProviderUtil.SplitPath(s1.Path.CurrentLocation.Path, s1.Drive.Current);
if (s1.Drive.Current is NosDBPSDriveInfo)
{
if (ConfigurationConnection.ConfigCluster == null)
{
throw new Exception(ProviderUtil.CONFIG_NOT_CONNECTED_EXCEPTION);
}
NodeDetail thisNode;
if (new NoSDbDetail(pathChunks, s1.Drive.Current).TryGetNodeDetail(out thisNode))
{
}
if (thisNode.NodeType.Equals(PathType.Collection))
{
_databaseName = ((CollectionValueDetail)thisNode).Database;
_collectionName = thisNode.NodeName;
switch (Format.ToLower())
{
case "json":
_dataType = EXIMDataType.JSON;
break;
case "csv":
_dataType = EXIMDataType.CSV;
break;
}
}
else
{
throw new Exception("You must be in NosDB:\\databasecluster\\databases\\database\\collections\\collection> for Export-Data.");
}
}
else
{
throw new Exception("You must be in NosDB:\\databasecluster\\databases\\database\\collections\\collection> for Export-Data.");
}
}
/// <summary> /// Reads data from a file and creates IJSONDOcuments against it /// </summary> /// <param name="dataType"></param> /// <param name="path"></param> /// <param name="collection"></param> /// <param name="database"></param> /// <returns></returns> public abstract IEnumerable <List <JSONDocument> > Read(EXIMDataType dataType, string path);
/// <summary> /// Write data in a predefined format against a given query at a specified path /// </summary> /// <param name="dataType"></param> /// <param name="path"></param> /// <param name="docList"></param> /// <returns></returns> public abstract RecoveryOperationStatus Write(EXIMDataType dataType, string path, string collection, string fileName, string database, List <IJSONDocument> docList);
public override RecoveryOperationStatus Write(EXIMDataType dataType, string path, string collection, string fileName, string database, List <Server.Engine.IJSONDocument> docList)
{
RecoveryOperationStatus state = base.ValidatePath(path, RecoveryJobType.Export);
if (state.Status == RecoveryStatus.Success)
{
try
{
bool first = true;
IDictionary <int, string> headerPositionMap = null;
string file = string.Empty;
if (!string.IsNullOrEmpty(fileName))
{
file = Path.Combine(path, fileName + _fileExtension);
}
else
{
string defaultName = database + "_" + collection;
file = Path.Combine(path, defaultName + _fileExtension);
}
using (StreamWriter writer = new StreamWriter(file))
{
if (docList.Count > 0)
{
foreach (JSONDocument document in docList)
{
if (first)
{
WriteHeaderRow(document, out headerPositionMap, writer);
first = false;
}
else
{
IDictionary <string, object> valueMap = GetCsvReadyValues(document);
string _delim = ",";
string vString = string.Empty;
int pos = 0;
foreach (int hPos in headerPositionMap.Keys)
{
if (pos != 0)
{
vString += _delim;
}
if (valueMap.ContainsKey(headerPositionMap[hPos]))
{
vString += valueMap[headerPositionMap[hPos]].ToString();
}
else
{
vString += _delim;
}
pos++;
}
writer.WriteLine(vString);
}
}
writer.Close();
}
}
}
catch (Exception exp)
{
if (LoggerManager.Instance.EXIMLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
{
LoggerManager.Instance.EXIMLogger.Error("CSVEXIMUtil.Export()", exp.ToString());
}
state.Status = RecoveryStatus.Failure;
state.Message = exp.ToString();
}
return(state);
}
else
{
return(state);
}
}
public override IEnumerable <List <JSONDocument> > Read(EXIMDataType dataType, string path)
{
List <JSONDocument> items = new List <JSONDocument>();
RecoveryOperationStatus state = base.ValidatePath(path, RecoveryJobType.Import);
if (state.Status == RecoveryStatus.Success)
{
if (ValidateExtension(path))
{
using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read))
using (StreamReader reader = new StreamReader(stream))
{
// if file is not empty
if (reader.Peek() > 0)
{
position = 0;
IDictionary <int, string> header = ReadHeaderRow(reader);
IDictionary <int, object> value = null;
while (!reader.EndOfStream)
{
value = ReadValueRow(reader);
if (header.Count != value.Count)
{
throw new InvalidDataException("Invalid data provided the number of columns is not equal to header row");
}
JSONDocument doc = CreateJSONDocument(header, value);
if (doc != null)
{
if (items.Count <= base.ChunkSize)
{
items.Add(doc);
}
else
{
items.Add(doc);
yield return(items);
items.Clear();
}
}
}
if (items.Count > 0)
{
yield return(items);
}
}
else
{
yield return(items);
}
}
}
else
{
throw new ArgumentException("Invalid file extension");
}
}
else
{
throw new ArgumentException("Invalid file path provided");
}
}
public RecoveryOperationStatus Import(string database, string collection, string path, bool updateMode, EXIMDataType dataType)
{
RecoveryOperationStatus state = new RecoveryOperationStatus(RecoveryStatus.Success);
EXIMBase eximBase = null;
switch (dataType)
{
case EXIMDataType.CSV:
eximBase = new CSVEXIMUtil();
break;
case EXIMDataType.JSON:
eximBase = new JSONEXIMUtil();
break;
}
if (eximBase != null)
{
try
{
foreach (List<JSONDocument> docList in eximBase.Read(dataType, path))
{
if (docList != null)
{
if (docList.Count > 0)
{
// create insert operation if mode is update perform replace operation on
List<FailedDocument> failedDoc = InsertDocuments(docList);
if (failedDoc != null)
{
if (failedDoc.Count > 0)
{
if (updateMode)
{
List<JSONDocument> retryList = new List<JSONDocument>();
foreach (FailedDocument failed in failedDoc)
{
foreach (JSONDocument orgDoc in docList)
{
if (orgDoc.Key.Equals(failed.DocumentKey))
{
retryList.Add(orgDoc);
}
}
}
failedDoc = ReplaceDocuments(retryList);
if (failedDoc != null)
{
if (failedDoc.Count > 0)
{
if (LoggerManager.Instance.EXIMLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
LoggerManager.Instance.EXIMLogger.Error("Import()", failedDoc.Count + "failed to import in collection" + collection);
state.Status = RecoveryStatus.Failure;
state.Message = failedDoc.Count + "failed to import in collection" + collection;
}
}
}
else
{
if (LoggerManager.Instance.EXIMLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
LoggerManager.Instance.EXIMLogger.Error("Import()", failedDoc.Count + "failed to import in collection" + collection);
state.Status = RecoveryStatus.Failure;
state.Message = failedDoc.Count + "failed to import in collection" + collection;
}
}
}
}
}
}
}
catch (Exception exp)
{
if (LoggerManager.Instance.EXIMLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
LoggerManager.Instance.EXIMLogger.Error("Import()", exp.ToString());
state.Status = RecoveryStatus.Failure;
state.Message = exp.Message.ToString() +" For details kindly review the log file";
}
}
return state;
}
/// <summary>
/// Exports IJSONDocuments to export file.
/// </summary>
/// <param name="collection"></param>
/// <param name="docList"></param>
/// <param name="path"></param>
/// <param name="dataType"></param>
/// <returns></returns>
public RecoveryOperationStatus Export(string collection, List<IJSONDocument> docList, string path,string fileName,string database, EXIMDataType dataType)
{
RecoveryOperationStatus state = new RecoveryOperationStatus(RecoveryStatus.Failure);
EXIMBase eximBase = null;
switch (dataType)
{
case EXIMDataType.CSV:
eximBase = new CSVEXIMUtil();
break;
case EXIMDataType.JSON:
eximBase = new JSONEXIMUtil();
break;
}
if (eximBase != null)
{
state = eximBase.Write(dataType, path, collection,fileName,database, docList);
}
return state;
}
/// <summary>
/// Exports data against a specified collection, incase no query is specified entire collection data will be dumped to file
/// </summary>
/// <param name="database"></param>
/// <param name="collection"></param>
/// <param name="query"></param>
/// <param name="path"></param>
/// <returns></returns>
public RecoveryOperationStatus Export(string database, string collection, string query,ICollection<IParameter> parameters, string path,string filename, EXIMDataType dataType)
{
RecoveryOperationStatus state = new RecoveryOperationStatus(RecoveryStatus.Failure);
try
{
//create query if not provided
if (!string.IsNullOrEmpty(collection))
{
if (string.IsNullOrEmpty(query))
{
if (!string.IsNullOrEmpty(collection))
{
query = "Select * from $" + collection + "$";
parameters = new List<IParameter>();
}
}
// query data from the collection
ICollectionReader reader = ExecuteReader(query, parameters);
List<IJSONDocument> exportList = new List<IJSONDocument>();
while (reader.ReadNext())
{
exportList.Add(reader.GetObject<IJSONDocument>());
}
//write json documents to the file
state = Export(collection, exportList, path,filename,database, dataType);
}
else
{
throw new ArgumentException("Invalid Collection provided");
}
}
catch(Exception exp)
{
if (LoggerManager.Instance.EXIMLogger != null && LoggerManager.Instance.RecoveryLogger.IsErrorEnabled)
LoggerManager.Instance.EXIMLogger.Error("Export()",exp.ToString());
state.Message = exp.ToString();
}
return state;
}