public async Task AddManyOperations(List <CardOperation> operations)
{
Throw.If.IsNull(operations, nameof(DbLogicManager), nameof(AddManyOperations), nameof(operations));
if (operations.Count > 0)
{
await _cardOperationStorage.AddMany(operations);
}
}
private void SaveOperationsToStorage(List <CardOperation> operations, CardOperationsImport import)
{
operations.ForEach(op => op.ImportId = import.Id);
_cardOperationStorage.AddMany(operations);
}
public override async Task Load(LoadFile request, IServerStreamWriter <QueryResponse> responseStream, ServerCallContext context)
{
if (request.Type == "nt")
{
Console.WriteLine($"Loading file: {request.Path}");
try
{
Stream data = null;
var TotalLength = 0L;
var TotalProgress = 0L;
var totalLengthDetermined = false;
if (!File.Exists(request.Path))
{
var wc = new WebClient();
var file = Path.GetTempFileName();
await wc.DownloadFileTaskAsync(request.Path, file);
data = File.OpenRead(file);
TotalLength = data.Length;
totalLengthDetermined = true;
}
else
{
data = File.OpenRead(request.Path);
TotalLength = data.Length;
totalLengthDetermined = true;
}
var timer = Stopwatch.StartNew();
await using var cleanup = data;
await using var bs = new BufferedStream(data, 4096);
{
var lastNewLinePosition = 0;
var memoryWritePos = 0L;
var unReadBytes = 0L;
var memory = new byte[81920];
await using var mainView = new MemoryStream(memory);
do
{
var newBytesAdded = await bs.ReadAsync(memory, (int)memoryWritePos, (int)(memory.Length - memoryWritePos));
if (!totalLengthDetermined)
{
TotalLength += newBytesAdded;
}
var newBytesEndPos = memoryWritePos + newBytesAdded;
if (newBytesAdded == 0)
{
break;
}
// move back to beginning
mainView.Seek(0, SeekOrigin.Begin);
// find good place to stop
//var endOfLastTriple = await ParseNTStreamNoTree(new MemoryStream(memory, 0,(int)newBytesEndPos));
var endOfLastTriple = ParseForNewLine(new Span <byte>(memory, 0, (int)newBytesEndPos));
// just incase we have have reached the end, we want to make sure we try to read the last bit...
if (endOfLastTriple == 0 && newBytesEndPos > 0)
{
unReadBytes = newBytesEndPos;
}
else
{
unReadBytes = newBytesEndPos - endOfLastTriple;
}
// create a vew up to that position;
var pageView = new MemoryStream(memory, 0, endOfLastTriple);
// run the parser on that pageView.
await ParseNTriplesStream(pageView);
// move the end of the stream that we didn't read back to the beginning
//Console.WriteLine($"BBC: endoflastTriple:{endOfLastTriple}, unReadBytes:{unReadBytes}, urb32:{Convert.ToInt32(unReadBytes)}");
Buffer.BlockCopy(memory, endOfLastTriple, memory, 0, Convert.ToInt32(unReadBytes));
memoryWritePos = unReadBytes;
TotalProgress += endOfLastTriple;
if (timer.Elapsed > TimeSpan.FromSeconds(1))
{
var qr = new QueryResponse();
qr.LoadFileResponse = new LoadFileResponse
{
Progress = TotalProgress,
Length = TotalLength
};
await responseStream.WriteAsync(qr);
}
if (context.CancellationToken.IsCancellationRequested)
{
Console.WriteLine($"Cancel requested for file load. {request.Path }");
break;
}
} while (unReadBytes > 0);
}
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
else if (request.Type == "graphml")
{
Console.WriteLine($"Loading file: {request.Path}");
try
{
var sw = Stopwatch.StartNew();
var nodes = TinkerPop.buildNodesFromFile(request.Path);
await _db.AddMany(nodes).ContinueWith(adding =>
{
if (adding.IsCompletedSuccessfully)
{
sw.Stop();
Console.WriteLine(
$"\nstatus> put done in {sw.ElapsedMilliseconds}ms");
}
else
{
Console.WriteLine(
$"\nstatus> put err({adding?.Exception?.InnerException?.Message})");
}
});
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
}
