public override QueryResult Import(IEnumerable<String> myLines, IGraphDBSession myIGraphDBSession, DBContext myDBContext, UInt32 parallelTasks = 1, IEnumerable<String> comments = null, ulong? offset = null, ulong? limit = null, VerbosityTypes verbosityTypes = VerbosityTypes.Errors)
{
var gqlQuery = new GraphQLQuery(myDBContext.DBPluginManager);
#region Evaluate Limit and Offset
if (offset != null)
{
myLines = myLines.SkipULong(offset.Value);
}
if (limit != null)
{
myLines = myLines.TakeULong(limit.Value);
}
#endregion
var queryResult = new QueryResult();
#region Import queries
if (parallelTasks > 1)
{
queryResult = ExecuteAsParallel(myLines, myIGraphDBSession, gqlQuery, verbosityTypes, parallelTasks, comments);
}
else
{
queryResult = ExecuteAsSingleThread(myLines, myIGraphDBSession, gqlQuery, verbosityTypes, comments);
}
#endregion
return queryResult;
}
public QueryResult Import(String location, IGraphDBSession graphDBSession, DBContext myDBContext, UInt32 parallelTasks = 1, IEnumerable<string> comments = null, UInt64? offset = null, UInt64? limit = null, VerbosityTypes verbosityType = VerbosityTypes.Errors)
{
IEnumerable<String> lines = null;
#region Read querie lines from location
try
{
if (location.ToLower().StartsWith(@"file:\\"))
{
lines = ReadFile(location.Substring(@"file:\\".Length));
}
else if (location.ToLower().StartsWith("http://"))
{
lines = ReadHttpResource(location);
}
else
{
return new QueryResult(new Exceptional(new Error_InvalidImportLocation(location, @"file:\\", "http://")));
}
}
catch (Exception ex)
{
return new QueryResult(new Exceptional(new Error_ImportFailed(ex)));
}
#endregion
#region Start import using the AGraphDBImport implementation
return Import(lines, graphDBSession, myDBContext, parallelTasks, comments, offset, limit, verbosityType);
#endregion
}
public JobLogger(
VerbosityTypes verbosity,
IConfiguration configuration
)
{
Verbosity = verbosity;
Configuration = configuration;
}
public void Init(ParsingContext context, ParseTreeNode parseNode)
{
var verbosityType = VerbosityType;
if (HasChildNodes(parseNode) && Enum.TryParse <VerbosityTypes>(parseNode.ChildNodes[1].Token.Text, true, out verbosityType))
{
VerbosityType = verbosityType;
}
}
public void GetContent(CompilerContext context, ParseTreeNode parseNode)
{
var verbosityType = VerbosityType;
if (parseNode.HasChildNodes() && Enum.TryParse<VerbosityTypes>(parseNode.ChildNodes[1].Token.Text, true, out verbosityType))
{
VerbosityType = verbosityType;
}
}
public void Init(ParsingContext context, ParseTreeNode parseNode)
{
var verbosityType = VerbosityType;
if (HasChildNodes(parseNode) && Enum.TryParse<VerbosityTypes>(parseNode.ChildNodes[1].Token.Text, true, out verbosityType))
{
VerbosityType = verbosityType;
}
}
public IEnumerable <IVertexView> Import(String myLocation,
IGraphDB myGraphDB,
IGraphQL myGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
UInt32 myParallelTasks = 1U,
IEnumerable <string> myComments = null,
UInt64?myOffset = null,
UInt64?myLimit = null,
VerbosityTypes myVerbosityType = VerbosityTypes.Silent,
Dictionary <string, string> myOptions = null)
{
Stream stream = null;
#region Read querie lines from location
try
{
#region file
if (myLocation.ToLower().StartsWith(@"file:\\"))
{
//lines = ReadFile(location.Substring(@"file:\\".Length));
stream = GetStreamFromFile(myLocation.Substring(@"file:\\".Length));
}
#endregion
#region http
else if (myLocation.ToLower().StartsWith("http://"))
{
stream = GetStreamFromHttp(myLocation);
}
#endregion
else
{
throw new InvalidImportLocationException(myLocation, @"file:\\", "http://");
}
#region Start import using the AGraphDBImport implementation and return the result
var result = Import(stream, myGraphDB, myGraphQL, mySecurityToken, myTransactionToken, myParallelTasks, myComments, myOffset, myLimit, myVerbosityType);
return(AggregateListOfListOfVertices(result));
#endregion
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
#endregion
}
public QueryResult Import(String myLocation, IGraphDB myGraphDB, IGraphQL myGraphQL, SecurityToken mySecurityToken, TransactionToken myTransactionToken, UInt32 myParallelTasks = 1U, IEnumerable<string> myComments = null, UInt64? myOffset = null, UInt64? myLimit = null, VerbosityTypes myVerbosityType = VerbosityTypes.Silent)
{
ASonesException error;
Stream stream = null;
QueryResult result;
#region Read querie lines from location
try
{
#region file
if (myLocation.ToLower().StartsWith(@"file:\\"))
{
//lines = ReadFile(location.Substring(@"file:\\".Length));
stream = GetStreamFromFile(myLocation.Substring(@"file:\\".Length));
}
#endregion
#region http
else if (myLocation.ToLower().StartsWith("http://"))
{
stream = GetStreamFromHttp(myLocation);
}
#endregion
else
{
error = new InvalidImportLocationException(myLocation, @"file:\\", "http://");
result = new QueryResult("", ImportFormat, 0L, ResultType.Failed, null, error);
return result;
}
#region Start import using the AGraphDBImport implementation and return the result
return Import(stream, myGraphDB, myGraphQL, mySecurityToken, myTransactionToken, myParallelTasks, myComments, myOffset, myLimit, myVerbosityType);
#endregion
}
catch (Exception ex)
{
#region throw new exception
error = new ImportFailedException(ex);
result = new QueryResult("", ImportFormat, 0L, ResultType.Failed, null, error);
return result;
#endregion
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
#endregion
}
public override Exceptional Export(DBContext myDBContext, IDumpable myGrammar, IEnumerable<TypeManagement.GraphDBType> myTypes, DumpTypes myDumpType, VerbosityTypes verbosityType = VerbosityTypes.Errors)
{
var dumpReadout = new Dictionary<String, Object>();
if ((myDumpType & DumpTypes.GDDL) == DumpTypes.GDDL)
{
var graphDDL = myGrammar.ExportGraphDDL(DumpFormats.GQL, myDBContext, myTypes);
if (!graphDDL.Success())
{
return new Exceptional(graphDDL);
}
//dumpReadout.Add("GDDL", graphDDL.Value);
var writeResult = Write(DumpTypes.GDDL, graphDDL.Value);
if (!writeResult.Success())
{
return writeResult;
}
}
if ((myDumpType & DumpTypes.GDML) == DumpTypes.GDML)
{
var graphDML = myGrammar.ExportGraphDML(DumpFormats.GQL, myDBContext, myTypes);
if (!graphDML.Success())
{
return new Exceptional(graphDML);
}
//dumpReadout.Add("GDML", _GraphDML.Value);
var writeResult = Write(DumpTypes.GDML, graphDML.Value);
if (!writeResult.Success())
{
return writeResult;
}
}
return Exceptional.OK;
}
public QueryResult Import(String location, IGraphDBSession graphDBSession, DBContext myDBContext, UInt32 parallelTasks = 1, IEnumerable<string> comments = null, UInt64? offset = null, UInt64? limit = null, VerbosityTypes verbosityType = VerbosityTypes.Errors)
{
QueryResult result = new QueryResult();
Stream stream = null;
#region Read querie lines from location
try
{
if (location.ToLower().StartsWith(@"file:\\"))
{
//lines = ReadFile(location.Substring(@"file:\\".Length));
stream = GetStreamFromFile(location.Substring(@"file:\\".Length));
}
else if (location.ToLower().StartsWith("http://"))
{
stream = GetStreamFromHttp(location);
}
else
{
return new QueryResult(new Exceptional(new Error_InvalidImportLocation(location, @"file:\\", "http://")));
}
#region Start import using the AGraphDBImport implementation and return the result
return Import(stream, graphDBSession, myDBContext, parallelTasks, comments, offset, limit, verbosityType);
#endregion
}
catch (Exception ex)
{
return new QueryResult(new Exceptional(new Error_ImportFailed(ex)));
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
#endregion
}
public QueryResult Export(String destination, DBContext myDBContext, IDumpable myGrammar, IEnumerable<String> myTypes, DumpTypes myDumpType, VerbosityTypes verbosityType = VerbosityTypes.Errors)
{
_Destination = destination;
#region Open destination
var openStreamResult = OpenStream(destination);
if (!openStreamResult.Success())
{
return new QueryResult(openStreamResult);
}
#endregion
#region Start export using the AGraphDBImport implementation
var result = Export(myDBContext, myGrammar, GetTypes(myDBContext, myTypes), myDumpType, verbosityType);
#endregion
#region Close destination
var closeStreamResult = CloseStream();
if (!closeStreamResult.Success())
{
return new QueryResult(openStreamResult);
}
#endregion
if (!result.Success())
{
return new QueryResult(result);
}
return new QueryResult(new List<Vertex>() { new Vertex(_DumpReadout) });
}
public override QueryResult Import(System.IO.Stream myInputStream, IGraphDBSession myIGraphDBSession, DBContext myDBContext, UInt32 myParallelTasks = 1, IEnumerable<String> myComments = null, ulong? myOffset = null, ulong? myLimit = null, VerbosityTypes myVerbosityType = VerbosityTypes.Errors)
{
var gqlQuery = new GraphQLQuery(myDBContext.DBPluginManager);
var lines = ReadLinesFromStream(myInputStream);
#region Evaluate Limit and Offset
if (myOffset != null)
{
lines = lines.SkipULong(myOffset.Value);
}
if (myLimit != null)
{
lines = lines.TakeULong(myLimit.Value);
}
#endregion
var queryResult = new QueryResult();
#region Import queries
if (myParallelTasks > 1)
{
queryResult = ExecuteAsParallel(lines, myIGraphDBSession, gqlQuery, myVerbosityType, myParallelTasks, myComments);
}
else
{
queryResult = ExecuteAsSingleThread(lines, myIGraphDBSession, gqlQuery, myVerbosityType, myComments);
}
#endregion
return queryResult;
}
private QueryResult ExecuteAsSingleThread(IEnumerable<String> myLines, IGraphQL myIGraphQL, SecurityToken mySecurityToken, TransactionToken myTransactionToken, VerbosityTypes myVerbosityType, IEnumerable<String> comments = null)
{
#region data
QueryResult queryResult = null;
Int64 numberOfLine = 0;
var aggregatedResults = new List<IEnumerable<IVertexView>>();
Stopwatch StopWatchLines = new Stopwatch();
#endregion
#region check lines and execute query
StopWatchLines.Reset();
StopWatchLines.Start();
foreach (var _Line in myLines)
{
numberOfLine++;
if (String.IsNullOrWhiteSpace(_Line))
{
continue;
}
#region Skip comments
if (IsComment(_Line, comments))
continue;
#endregion
#region execute query
var tempResult = myIGraphQL.Query(mySecurityToken, myTransactionToken, _Line);
#endregion
#region Add errors and break execution
if (tempResult.TypeOfResult == ResultType.Failed)
{
if (tempResult.Error.Message.Equals("Mal-formed string literal - cannot find termination symbol."))
Debug.WriteLine("Query at line [" + numberOfLine + "] [" + _Line + "] failed with " + tempResult.Error.ToString() + " add next line...");
if (myVerbosityType == VerbosityTypes.Errors)
{
queryResult = new QueryResult(_Line, ImportFormat, 0L, ResultType.Failed, tempResult.Vertices, tempResult.Error);
break;
}
}
aggregatedResults.Add(tempResult.Vertices);
#endregion
}
StopWatchLines.Stop();
#endregion
//add the results of each query into the queryResult
if(queryResult != null)
queryResult = new QueryResult(myLines.ToString(), ImportFormat, Convert.ToUInt64(StopWatchLines.ElapsedMilliseconds), queryResult.TypeOfResult, AggregateListOfListOfVertices(aggregatedResults), queryResult.Error);
else
queryResult = new QueryResult(myLines.ToString(), ImportFormat, Convert.ToUInt64(StopWatchLines.ElapsedMilliseconds), ResultType.Successful, AggregateListOfListOfVertices(aggregatedResults));
return queryResult;
}
private QueryResult ExecuteAsParallel(IEnumerable<String> myLines, IGraphQL myIGraphQL, SecurityToken mySecurityToken, TransactionToken myTransactionToken, VerbosityTypes myVerbosityType, UInt32 myParallelTasks = 1U, IEnumerable<String> comments = null)
{
#region data
QueryResult queryResult = new QueryResult(myLines.ToString(), ImportFormat, 0L, ResultType.Successful);
Int64 numberOfLine = 0;
var query = String.Empty;
var aggregatedResults = new List<IEnumerable<IVertexView>>();
Stopwatch StopWatchLine = new Stopwatch();
Stopwatch StopWatchLines = new Stopwatch();
#endregion
#region Create parallel options
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = (int)myParallelTasks
};
#endregion
#region check lines and execute query
StopWatchLines.Start();
Parallel.ForEach(myLines, parallelOptions, (line, state) =>
{
if (!IsComment(line, comments))
{
Interlocked.Add(ref numberOfLine, 1L);
if (!IsComment(line, comments)) // Skip comments
{
var qresult = ExecuteQuery(line, myIGraphQL, mySecurityToken, myTransactionToken);
#region VerbosityTypes.Full: Add result
if (myVerbosityType == VerbosityTypes.Full)
{
lock (aggregatedResults)
{
aggregatedResults.Add(qresult.Vertices);
}
}
#endregion
#region !VerbosityTypes.Silent: Add errors and break execution
if (qresult.TypeOfResult != ResultType.Successful && myVerbosityType != VerbosityTypes.Silent)
{
queryResult = new QueryResult(line, ImportFormat, Convert.ToUInt64(StopWatchLine.ElapsedMilliseconds), ResultType.Failed, qresult.Vertices, qresult.Error);
state.Break();
}
#endregion
}
}
});
StopWatchLines.Stop();
#endregion
//add the results of each query into the queryResult
if (queryResult != null)
queryResult = new QueryResult(myLines.ToString(), ImportFormat, Convert.ToUInt64(StopWatchLines.ElapsedMilliseconds), queryResult.TypeOfResult, AggregateListOfListOfVertices(aggregatedResults), queryResult.Error);
else
queryResult = new QueryResult(myLines.ToString(), ImportFormat, Convert.ToUInt64(StopWatchLines.ElapsedMilliseconds), ResultType.Successful, AggregateListOfListOfVertices(aggregatedResults));
return queryResult;
}
public IEnumerable<IVertexView> Import(String myLocation,
IGraphDB myGraphDB,
IGraphQL myGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
UInt32 myParallelTasks = 1U,
IEnumerable<string> myComments = null,
UInt64? myOffset = null,
UInt64? myLimit = null,
VerbosityTypes myVerbosityType = VerbosityTypes.Silent,
Dictionary<string, string> myOptions = null)
{
if (myGraphDB == null)
throw new ArgumentNullException("myGraphDB");
if (myGraphQL == null)
throw new ArgumentNullException("myGraphQL");
if (!typeof(SonesGraphDB).Equals(myGraphDB.GetType()))
throw new ArgumentException("FASTIMPORT is designed for SonesGraphDB only.");
lock (_lock)
{
//if verbositiy is silent, we do not configure the logger, so it is an empty logger.
if (myVerbosityType != VerbosityTypes.Silent)
{
Level logLevel = (myVerbosityType == VerbosityTypes.Full)
? Level.FINE
: Level.INFO;
LogManager.Instance.ConfigureLogger("FastImport", new FileLogger(_logpath, logLevel));
}
//store some arguments as fields, because there is at most one execution at any time.
_logger = LogManager.Instance.GetLogger("FastImport");
_db = myGraphDB;
_ql = myGraphQL;
_security = mySecurityToken;
_transaction = myTransactionToken;
_offset = myOffset;
_limit = myLimit;
_currentImport = 0L;
_closed = false;
_autoCreateIncomingEdges = false;
if (myOptions != null)
{
if (myOptions.ContainsKey("AutoCreateIncomingEdges"))
bool.TryParse(myOptions["AutoCreateIncomingEdges"], out _autoCreateIncomingEdges);
}
if (!_autoCreateIncomingEdges)
_sorter = new IncomingEdgeSorter(myOptions);
try
{
if (myLocation.ToLower().StartsWith(@"file:\\"))
{
stream = GetStreamFromFile(myLocation.Substring(@"file:\\".Length));
}
else
{
_logger.Log(Level.SEVERE, "Location does not start with file:\\\\.");
throw new InvalidImportLocationException(myLocation, @"file:\\");
}
#region Start import using the AGraphDBImport implementation and return the result
return Import();
#endregion
}
catch (Exception ex)
{
//if something unexpected happens we log it and return a query result with failed.
_logger.Log(Level.SEVERE, "Exception thrown:\n", ex);
throw new ImportFailedException(ex);
}
finally
{
if (stream != null)
{
_logger.Log(Level.FINE, "Stream closed");
stream.Dispose();
if (_sorter != null)
_sorter.Dispose();
}
}
}
}
public override void GetContent(CompilerContext context, ParseTreeNode parseNode)
{
//var dbContext = context.IContext as DBContext;
// parseNode.ChildNodes[0] - import symbol
// parseNode.ChildNodes[1] - from symbol
SourceLocation = parseNode.ChildNodes[2].Token.ValueString;
// parseNode.ChildNodes[3] - format symbol
ImportFormat = parseNode.ChildNodes[4].Token.Text;
ParallelTasks = (parseNode.ChildNodes[5].AstNode as ParallelTasksNode).ParallelTasks;
Comments = (parseNode.ChildNodes[6].AstNode as CommentsNode).Comments;
Offset = (parseNode.ChildNodes[7].AstNode as OffsetNode).Count;
Limit = (parseNode.ChildNodes[8].AstNode as LimitNode).Count;
VerbosityType = (parseNode.ChildNodes[9].AstNode as VerbosityNode).VerbosityType;
}
public VerbosityNode()
{
VerbosityType = VerbosityTypes.Errors;
}
public abstract QueryResult Import(Stream myInputStream, IGraphDBSession myGraphDBSession, DBContext myDBContext, UInt32 myParallelTasks = 1, IEnumerable<string> myComments = null, UInt64? myOffset = null, UInt64? myLimit = null, VerbosityTypes myVerbosityType = VerbosityTypes.Errors);
private QueryResult ExecuteAsSingleThread(IEnumerable<String> myLines, IGraphDBSession myIGraphDBSession, GraphQLQuery myGQLQuery, VerbosityTypes verbosityTypes, IEnumerable<String> comments = null)
{
var queryResult1 = new QueryResult();
Int64 numberOfLine = 0;
var query = String.Empty;
var aggregatedResults = new List<IEnumerable<Vertex>>();
foreach (var _Line in myLines)
{
numberOfLine++;
#region Skip comments
if (IsComment(_Line, comments))
{
continue;
}
#endregion
query += _Line;
var qresult = ExecuteQuery(query, myIGraphDBSession, myGQLQuery);
#region VerbosityTypes.Full: Add result
if (verbosityTypes == VerbosityTypes.Full)
{
aggregatedResults.Add(qresult.Vertices);
}
#endregion
#region !VerbosityTypes.Silent: Add errors and break execution
if (qresult.ResultType == ResultType.Failed)
{
if (qresult.Errors.Any(e => (e is Error_GqlSyntax) && (e as Error_GqlSyntax).SyntaxErrorMessage.Equals("Mal-formed string literal - cannot find termination symbol.")))
{
Debug.WriteLine("Query at line [" + numberOfLine + "] [" + query + "] failed with " + qresult.GetIErrorsAsString() + " add next line...");
continue;
}
if (verbosityTypes != VerbosityTypes.Silent)
{
queryResult1.PushIError(new Error_ImportFailed(query, numberOfLine));
queryResult1.PushIErrors(qresult.Errors);
queryResult1.PushIWarnings(qresult.Warnings);
}
break;
}
else if (qresult.ResultType == ResultType.PartialSuccessful && verbosityTypes != VerbosityTypes.Silent)
{
queryResult1.PushIWarning(new Warning_ImportWarning(query, numberOfLine));
queryResult1.PushIWarnings(qresult.Warnings);
}
#endregion
query = String.Empty;
}
//add the results of each query into the queryResult
queryResult1.Vertices = AggregateListOfListOfVertices(aggregatedResults);
return queryResult1;
}
private IEnumerable<IEnumerable<IVertexView>> ExecuteAsSingleThread(IEnumerable<String> myLines,
IGraphQL myIGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
VerbosityTypes myVerbosityType,
IEnumerable<String> comments = null)
{
#region data
Int64 numberOfLine = 0;
var aggregatedResults = new List<IEnumerable<IVertexView>>();
#endregion
#region check lines and execute query
foreach (var _Line in myLines)
{
numberOfLine++;
if (String.IsNullOrWhiteSpace(_Line))
{
continue;
}
#region Skip comments
if (IsComment(_Line, comments))
continue;
#endregion
#region execute query
var tempResult = myIGraphQL.Query(mySecurityToken, myTransactionToken, _Line);
#endregion
#region Add errors and break execution
if (tempResult.TypeOfResult == ResultType.Failed)
{
if (tempResult.Error.Message.Equals("Mal-formed string literal - cannot find termination symbol."))
Debug.WriteLine("Query at line [" + numberOfLine + "] [" + _Line + "] failed with " + tempResult.Error.ToString() + " add next line...");
if (myVerbosityType == VerbosityTypes.Errors)
{
if (tempResult.Error != null)
throw tempResult.Error;
else
throw new ImportFailedException(new UnknownException());
}
}
aggregatedResults.Add(tempResult.Vertices);
#endregion
}
return aggregatedResults;
#endregion
}
private IEnumerable <IEnumerable <IVertexView> > ExecuteAsSingleThread(IEnumerable <String> myLines,
IGraphQL myIGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
VerbosityTypes myVerbosityType,
IEnumerable <String> comments = null)
{
#region data
Int64 numberOfLine = 0;
var aggregatedResults = new List <IEnumerable <IVertexView> >();
#endregion
#region check lines and execute query
foreach (var _Line in myLines)
{
numberOfLine++;
if (String.IsNullOrWhiteSpace(_Line))
{
continue;
}
#region Skip comments
if (IsComment(_Line, comments))
{
continue;
}
#endregion
#region execute query
var tempResult = myIGraphQL.Query(mySecurityToken, myTransactionToken, _Line);
#endregion
#region Add errors and break execution
if (tempResult.TypeOfResult == ResultType.Failed)
{
if (tempResult.Error.Message.Equals("Mal-formed string literal - cannot find termination symbol."))
{
Debug.WriteLine("Query at line [" + numberOfLine + "] [" + _Line + "] failed with " + tempResult.Error.ToString() + " add next line...");
}
if (myVerbosityType == VerbosityTypes.Errors)
{
if (tempResult.Error != null)
{
throw tempResult.Error;
}
else
{
throw new ImportFailedException(new UnknownException());
}
}
}
aggregatedResults.Add(tempResult.Vertices);
#endregion
}
return(aggregatedResults);
#endregion
}
public abstract Exceptional Export(DBContext myDBContext, IDumpable myGrammar, IEnumerable<GraphDBType> myTypes, DumpTypes myDumpType, VerbosityTypes verbosityType = VerbosityTypes.Errors);
private void ExecuteAsSingleThread(IEnumerable <String> myLines,
IGraphQL myIGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
VerbosityTypes myVerbosityType,
IEnumerable <String> comments = null)
{
_logger.Log(Level.INFO, "Started GQL import.");
#region data
Int64 numberOfLine = 0;
#endregion
#region check lines and execute query
Stopwatch sw = new Stopwatch();
foreach (var _Line in myLines)
{
numberOfLine++;
if (String.IsNullOrWhiteSpace(_Line))
{
continue;
}
#region Skip comments
if (IsComment(_Line, comments))
{
continue;
}
#endregion
#region execute query
_logger.Log(Level.INFO, String.Format("Executing query {0}: {1}", _numberOfStatements, _Line));
sw.Reset();
sw.Start();
var tempResult = myIGraphQL.Query(mySecurityToken, myTransactionToken, _Line);
Interlocked.Increment(ref _numberOfStatements);
sw.Stop();
_logger.Log(Level.INFO, String.Format("Took {0}ms.", sw.Elapsed.TotalMilliseconds));
#endregion
#region Add errors and break execution
if (tempResult.TypeOfResult == ResultType.Failed)
{
if (tempResult.Error.Message.Equals("Mal-formed string literal - cannot find termination symbol."))
{
string error = "Query at line [" + numberOfLine + "] [" + _Line + "] failed with " + tempResult.Error.ToString() + " add next line...";
Debug.WriteLine(error);
_logger.Log(Level.WARNING, error);
}
if (myVerbosityType == VerbosityTypes.Errors)
{
if (tempResult.Error != null)
{
_logger.Log(Level.SEVERE, tempResult.Error.ToString());
}
else
{
_logger.Log(Level.SEVERE, "Import failed because of unknown exception");
}
}
}
#endregion
}
#endregion
_logger.Log(Level.INFO, String.Format("Finished GQL import and executed {0} statements", _numberOfStatements));
}
public abstract QueryResult Import(IEnumerable<String> lines, IGraphDBSession graphDBSession, DBContext myDBContext, UInt32 parallelTasks = 1, IEnumerable<string> comments = null, UInt64? offset = null, UInt64? limit = null, VerbosityTypes verbosityType = VerbosityTypes.Errors);
public IEnumerable<IVertexView> Import(String myLocation,
IGraphDB myGraphDB,
IGraphQL myGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
UInt32 myParallelTasks = 1U,
IEnumerable<string> myComments = null,
UInt64? myOffset = null,
UInt64? myLimit = null,
VerbosityTypes myVerbosityType = VerbosityTypes.Silent,
Dictionary<string, string> myOptions = null)
{
Stream stream = null;
#region Read querie lines from location
try
{
//if verbositiy is silent, we do not configure the logger, so it is an empty logger.
if (myVerbosityType != VerbosityTypes.Silent) {
Level logLevel = (myVerbosityType == VerbosityTypes.Full)
? Level.FINE
: Level.INFO;
LogManager.Instance.ConfigureLogger ("GqlImport", new FileLogger (_logpath, logLevel));
}
//store some arguments as fields, because there is at most one execution at any time.
_logger = LogManager.Instance.GetLogger ("GqlImport");
#region file
if (myLocation.ToLower().StartsWith(@"file:\\"))
{
//lines = ReadFile(location.Substring(@"file:\\".Length));
stream = GetStreamFromFile(myLocation.Substring(@"file:\\".Length));
}
#endregion
#region http
else if (myLocation.ToLower().StartsWith("http://"))
{
stream = GetStreamFromHttp(myLocation);
}
#endregion
else
{
throw new InvalidImportLocationException(myLocation, @"file:\\", "http://");
}
#region Start import using the AGraphDBImport implementation and return the result
Import(stream, myGraphDB, myGraphQL, mySecurityToken, myTransactionToken, myParallelTasks, myComments, myOffset, myLimit, myVerbosityType);
return new VertexView(new Dictionary<string, object> { { "Number of import statements:", _numberOfStatements } }, null).SingleEnumerable();
#endregion
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
#endregion
}
private QueryResult Import(Stream myInputStream, IGraphDB myIGraphDB, IGraphQL myGraphQL, SecurityToken mySecurityToken, TransactionToken myTransactionToken, UInt32 myParallelTasks = 1U, IEnumerable<string> myComments = null, ulong? myOffset = null, ulong? myLimit = null, VerbosityTypes myVerbosityType = VerbosityTypes.Silent)
{
var lines = ReadLinesFromStream(myInputStream);
#region Evaluate Limit and Offset
if (myOffset != null)
{
if(lines != null)
lines = lines.Skip<String>(Convert.ToInt32(myOffset.Value));
}
if (myLimit != null)
{
if(lines != null)
lines = lines.Take<String>(Convert.ToInt32(myLimit.Value));
}
#endregion
#region Import queries
QueryResult queryResult;
if (myParallelTasks > 1)
{
queryResult = ExecuteAsParallel(lines, myGraphQL, mySecurityToken, myTransactionToken, myVerbosityType, myParallelTasks, myComments);
}
else
{
queryResult = ExecuteAsSingleThread(lines, myGraphQL, mySecurityToken, myTransactionToken, myVerbosityType, myComments);
}
#endregion
return queryResult;
}
public IEnumerable<IVertexView> Import(String myLocation,
IGraphDB myGraphDB,
IGraphQL myGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
UInt32 myParallelTasks = 1U,
IEnumerable<string> myComments = null,
UInt64? myOffset = null,
UInt64? myLimit = null,
VerbosityTypes myVerbosityType = VerbosityTypes.Silent,
Dictionary<string, string> myOptions = null)
{
Stream stream = null;
#region Read querie lines from location
try
{
#region file
if (myLocation.ToLower().StartsWith(@"file:\\"))
{
//lines = ReadFile(location.Substring(@"file:\\".Length));
stream = GetStreamFromFile(myLocation.Substring(@"file:\\".Length));
}
#endregion
#region http
else if (myLocation.ToLower().StartsWith("http://"))
{
stream = GetStreamFromHttp(myLocation);
}
#endregion
else
{
throw new InvalidImportLocationException(myLocation, @"file:\\", "http://");
}
#region Start import using the AGraphDBImport implementation and return the result
var result = Import(stream, myGraphDB, myGraphQL, mySecurityToken, myTransactionToken, myParallelTasks, myComments, myOffset, myLimit, myVerbosityType);
return AggregateListOfListOfVertices(result);
#endregion
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
#endregion
}
private QueryResult ExecuteAsParallel(IEnumerable<String> myLines, IGraphDBSession myIGraphDBSession, GraphQLQuery myGQLQuery, VerbosityTypes verbosityTypes, UInt32 parallelTasks = 1, IEnumerable<String> comments = null)
{
var queryResult = new QueryResult();
var aggregatedResults = new List<IEnumerable<Vertex>>();
#region Create parallel options
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = (int)parallelTasks
};
#endregion
Int64 numberOfLine = 0;
Parallel.ForEach(myLines, parallelOptions, (line, state) =>
{
if (!IsComment(line, comments))
{
Interlocked.Add(ref numberOfLine, 1L);
if (!IsComment(line, comments)) // Skip comments
{
var qresult = ExecuteQuery(line, myIGraphDBSession, myGQLQuery);
#region VerbosityTypes.Full: Add result
if (verbosityTypes == VerbosityTypes.Full)
{
lock (aggregatedResults)
{
aggregatedResults.Add(qresult.Vertices);
}
}
#endregion
#region !VerbosityTypes.Silent: Add errors and break execution
if (qresult.ResultType != ResultType.Successful && verbosityTypes != VerbosityTypes.Silent)
{
lock (queryResult)
{
queryResult.PushIErrors(new[] { new Errors.Error_ImportFailed(line, numberOfLine) });
queryResult.PushIErrors(qresult.Errors);
queryResult.PushIWarnings(qresult.Warnings);
}
state.Break();
}
#endregion
}
}
});
//add the results of each query into the queryResult
queryResult.Vertices = AggregateListOfListOfVertices(aggregatedResults);
return queryResult;
}
private void ExecuteAsSingleThread(IEnumerable<String> myLines,
IGraphQL myIGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
VerbosityTypes myVerbosityType,
IEnumerable<String> comments = null)
{
_logger.Log (Level.INFO, "Started GQL import.");
#region data
Int64 numberOfLine = 0;
#endregion
#region check lines and execute query
Stopwatch sw = new Stopwatch ();
foreach (var _Line in myLines) {
numberOfLine++;
if (String.IsNullOrWhiteSpace (_Line)) {
continue;
}
#region Skip comments
if (IsComment (_Line, comments))
continue;
#endregion
#region execute query
_logger.Log (Level.INFO, String.Format ("Executing query {0}: {1}", _numberOfStatements, _Line));
sw.Reset ();
sw.Start ();
var tempResult = myIGraphQL.Query (mySecurityToken, myTransactionToken, _Line);
Interlocked.Increment (ref _numberOfStatements);
sw.Stop ();
_logger.Log (Level.INFO, String.Format ("Took {0}ms.", sw.Elapsed.TotalMilliseconds));
#endregion
#region Add errors and break execution
if (tempResult.TypeOfResult == ResultType.Failed) {
if (tempResult.Error.Message.Equals ("Mal-formed string literal - cannot find termination symbol.")) {
string error = "Query at line [" + numberOfLine + "] [" + _Line + "] failed with " + tempResult.Error.ToString () + " add next line...";
Debug.WriteLine (error);
_logger.Log (Level.WARNING, error);
}
if (myVerbosityType == VerbosityTypes.Errors) {
if (tempResult.Error != null) {
_logger.Log (Level.SEVERE, tempResult.Error.ToString ());
} else {
_logger.Log (Level.SEVERE, "Import failed because of unknown exception");
}
}
}
#endregion
}
#endregion
_logger.Log (Level.INFO, String.Format ("Finished GQL import and executed {0} statements", _numberOfStatements));
}
public VerbosityNode()
{
VerbosityType = VerbosityTypes.Errors;
}
public IEnumerable<IVertexView> Import(string myLocation,
IGraphDB myGraphDB,
IGraphQL myGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
UInt32 myParallelTasks = 1U,
IEnumerable<string> myComments = null,
UInt64? myOffset = null,
UInt64? myLimit = null,
VerbosityTypes myVerbosityTypes = VerbosityTypes.Silent,
Dictionary<string, string> myOptions = null)
{
#region data
if (myGraphDB == null)
{
throw new UnknownException(new ArgumentNullException("Missing GraphDB object"));
}
if(myLocation == null)
{
throw new UnknownException(new ArgumentNullException("Missing Location object"));
}
// if(mySecurityToken == null)
// {
// return new QueryResult("",
// ImportFormat,
// 0,
// ResultType.Failed,
// null,
// new UnknownException(new ArgumentNullException("mySecurityToken")));
// }
// if(myTransactionToken == null)
// {
// return new QueryResult("",
// ImportFormat,
// 0,
// ResultType.Failed,
// null,
// new UnknownException(new ArgumentNullException("myTransactionToken")));
// }
if(myOptions == null)
{
throw new UnknownException(
new ArgumentNullException(
String.Format("Missing Options {0}, {1}",
PARAM_VERTEXTYPENAME,
PARAM_EDGENAME
)));
}
_GraphDB = myGraphDB;
_SecurityToken = mySecurityToken;
_TransactionToken = myTransactionToken;
InitVertexSettings(myOptions);
var sw = new Stopwatch();
Stream stream = null;
#endregion
#region create vertex type
CreateVertexType();
#endregion
#region import
lock (_Lock)
{
#region read elements
try
{
if(myLocation.ToLower().StartsWith("file://") || myLocation.ToLower().StartsWith("file:\\"))
{
stream = GetStreamFromFile(myLocation.Substring("file://".Length));
}
else if(myLocation.StartsWith("http://"))
{
stream = GetStreamFromHttp(myLocation);
}
else
{
throw new FormatException("Given file location is invalid.");
}
var reader = XmlReader.Create(stream);
sw.Start();
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element)
{
switch (reader.Name)
{
case GraphMLTokens.GRAPHML:
break;
case GraphMLTokens.GRAPH:
break;
case GraphMLTokens.KEY:
ReadAttributeDefinition(reader);
break;
case GraphMLTokens.VERTEX:
ReadVertex(reader);
break;
case GraphMLTokens.EDGE:
ReadEdge(reader);
break;
default:
throw new XmlException(String.Format("Unsupported Node Type in GraphML File: {0}",
reader.Name));
}
}
}
}
catch (Exception ex)
{
// drop vertex type in case of exception
DropVertexType();
throw;
}
finally
{
if(stream != null)
{
stream.Close();
}
sw.Stop();
}
#endregion
}
#endregion
return null;
}
private void Import(Stream myInputStream, IGraphDB myIGraphDB, IGraphQL myGraphQL, SecurityToken mySecurityToken, Int64 myTransactionToken, UInt32 myParallelTasks = 1U, IEnumerable <string> myComments = null, ulong?myOffset = null, ulong?myLimit = null, VerbosityTypes myVerbosityType = VerbosityTypes.Silent)
{
var lines = ReadLinesFromStream(myInputStream);
#region Evaluate Limit and Offset
if (myOffset != null)
{
if (lines != null)
{
lines = lines.Skip <String> (Convert.ToInt32(myOffset.Value));
}
}
if (myLimit != null)
{
if (lines != null)
{
lines = lines.Take <String> (Convert.ToInt32(myLimit.Value));
}
}
#endregion
#region Import queries
ExecuteAsSingleThread(lines, myGraphQL, mySecurityToken, myTransactionToken, myVerbosityType, myComments);
#endregion
}
public DatabaseLogger(
VerbosityTypes verbosity,
IConfiguration configuration
) : base(verbosity, configuration)
{
}
public IEnumerable <IVertexView> Import(String myLocation,
IGraphDB myGraphDB,
IGraphQL myGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
UInt32 myParallelTasks = 1U,
IEnumerable <string> myComments = null,
UInt64?myOffset = null,
UInt64?myLimit = null,
VerbosityTypes myVerbosityType = VerbosityTypes.Silent,
Dictionary <string, string> myOptions = null)
{
Stream stream = null;
#region Read querie lines from location
try
{
//if verbositiy is silent, we do not configure the logger, so it is an empty logger.
if (myVerbosityType != VerbosityTypes.Silent)
{
Level logLevel = (myVerbosityType == VerbosityTypes.Full)
? Level.FINE
: Level.INFO;
LogManager.Instance.ConfigureLogger("GqlImport", new FileLogger(_logpath, logLevel));
}
//store some arguments as fields, because there is at most one execution at any time.
_logger = LogManager.Instance.GetLogger("GqlImport");
#region file
if (myLocation.ToLower().StartsWith(@"file:\\"))
{
//lines = ReadFile(location.Substring(@"file:\\".Length));
stream = GetStreamFromFile(myLocation.Substring(@"file:\\".Length));
}
#endregion
#region http
else if (myLocation.ToLower().StartsWith("http://"))
{
stream = GetStreamFromHttp(myLocation);
}
#endregion
else
{
throw new InvalidImportLocationException(myLocation, @"file:\\", "http://");
}
#region Start import using the AGraphDBImport implementation and return the result
Import(stream, myGraphDB, myGraphQL, mySecurityToken, myTransactionToken, myParallelTasks, myComments, myOffset, myLimit, myVerbosityType);
return(new VertexView(new Dictionary <string, object> {
{ "Number of import statements:", _numberOfStatements }
}, null).SingleEnumerable());
#endregion
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
#endregion
}
public IEnumerable <IVertexView> Import(string myLocation,
IGraphDB myGraphDB,
IGraphQL myGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
UInt32 myParallelTasks = 1U,
IEnumerable <string> myComments = null,
UInt64?myOffset = null,
UInt64?myLimit = null,
VerbosityTypes myVerbosityTypes = VerbosityTypes.Silent,
Dictionary <string, string> myOptions = null)
{
#region data
if (myGraphDB == null)
{
throw new UnknownException(new ArgumentNullException("Missing GraphDB object"));
}
if (myLocation == null)
{
throw new UnknownException(new ArgumentNullException("Missing Location object"));
}
// if(mySecurityToken == null)
// {
// return new QueryResult("",
// ImportFormat,
// 0,
// ResultType.Failed,
// null,
// new UnknownException(new ArgumentNullException("mySecurityToken")));
// }
// if(myTransactionToken == null)
// {
// return new QueryResult("",
// ImportFormat,
// 0,
// ResultType.Failed,
// null,
// new UnknownException(new ArgumentNullException("myTransactionToken")));
// }
if (myOptions == null)
{
throw new UnknownException(
new ArgumentNullException(
String.Format("Missing Options {0}, {1}",
PARAM_VERTEXTYPENAME,
PARAM_EDGENAME
)));
}
_GraphDB = myGraphDB;
_SecurityToken = mySecurityToken;
_TransactionToken = myTransactionToken;
InitVertexSettings(myOptions);
var sw = new Stopwatch();
Stream stream = null;
#endregion
#region create vertex type
CreateVertexType();
#endregion
#region import
lock (_Lock)
{
#region read elements
try
{
if (myLocation.ToLower().StartsWith("file://") || myLocation.ToLower().StartsWith("file:\\"))
{
stream = GetStreamFromFile(myLocation.Substring("file://".Length));
}
else if (myLocation.StartsWith("http://"))
{
stream = GetStreamFromHttp(myLocation);
}
else
{
throw new FormatException("Given file location is invalid.");
}
var reader = XmlReader.Create(stream);
sw.Start();
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element)
{
switch (reader.Name)
{
case GraphMLTokens.GRAPHML:
break;
case GraphMLTokens.GRAPH:
break;
case GraphMLTokens.KEY:
ReadAttributeDefinition(reader);
break;
case GraphMLTokens.VERTEX:
ReadVertex(reader);
break;
case GraphMLTokens.EDGE:
ReadEdge(reader);
break;
default:
throw new XmlException(String.Format("Unsupported Node Type in GraphML File: {0}",
reader.Name));
}
}
}
}
catch (Exception ex)
{
// drop vertex type in case of exception
DropVertexType();
throw;
}
finally
{
if (stream != null)
{
stream.Close();
}
sw.Stop();
}
#endregion
}
#endregion
return(null);
}
private IEnumerable<IEnumerable<IVertexView>> ExecuteAsParallel(IEnumerable<String> myLines,
IGraphQL myIGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
VerbosityTypes myVerbosityType,
UInt32 myParallelTasks = 1U,
IEnumerable<String> comments = null)
{
#region data
Int64 numberOfLine = 0;
var query = String.Empty;
var aggregatedResults = new List<IEnumerable<IVertexView>>();
#endregion
#region Create parallel options
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = (int)myParallelTasks
};
#endregion
#region check lines and execute query
Parallel.ForEach(myLines, parallelOptions, (line, state) =>
{
if (!IsComment(line, comments))
{
Interlocked.Increment(ref numberOfLine);
var qresult = ExecuteQuery(line, myIGraphQL, mySecurityToken, myTransactionToken);
#region VerbosityTypes.Full: Add result
if (myVerbosityType == VerbosityTypes.Full)
{
lock (aggregatedResults)
{
aggregatedResults.Add(qresult.Vertices);
}
}
#endregion
#region !VerbosityTypes.Silent: Add errors and break execution
if (qresult.TypeOfResult != ResultType.Successful && myVerbosityType != VerbosityTypes.Silent)
{
if (qresult.Error != null)
throw qresult.Error;
else
throw new ImportFailedException(new UnknownException());
}
#endregion
}
});
return aggregatedResults;
#endregion
}
private IEnumerable <IEnumerable <IVertexView> > ExecuteAsParallel(IEnumerable <String> myLines,
IGraphQL myIGraphQL,
SecurityToken mySecurityToken,
Int64 myTransactionToken,
VerbosityTypes myVerbosityType,
UInt32 myParallelTasks = 1U,
IEnumerable <String> comments = null)
{
#region data
Int64 numberOfLine = 0;
var query = String.Empty;
var aggregatedResults = new List <IEnumerable <IVertexView> >();
#endregion
#region Create parallel options
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = (int)myParallelTasks
};
#endregion
#region check lines and execute query
Parallel.ForEach(myLines, parallelOptions, (line, state) =>
{
if (!IsComment(line, comments))
{
Interlocked.Increment(ref numberOfLine);
var qresult = ExecuteQuery(line, myIGraphQL, mySecurityToken, myTransactionToken);
#region VerbosityTypes.Full: Add result
if (myVerbosityType == VerbosityTypes.Full)
{
lock (aggregatedResults)
{
aggregatedResults.Add(qresult.Vertices);
}
}
#endregion
#region !VerbosityTypes.Silent: Add errors and break execution
if (qresult.TypeOfResult != ResultType.Successful && myVerbosityType != VerbosityTypes.Silent)
{
if (qresult.Error != null)
{
throw qresult.Error;
}
else
{
throw new ImportFailedException(new UnknownException());
}
}
#endregion
}
});
return(aggregatedResults);
#endregion
}
