public static void UploadLargeStream(string target, string source, Action <AdsAnswer.Logging.EventType, string> writeLog = null, bool lineBoundary = false)
{
var fileSize = new System.IO.FileInfo(source).Length;
if (fileSize < 500 * 1024 * 1024) // <500M
{
UploadStream(target, source, lineBoundary);
return;
}
// split into blocks
List <long> blockOffsets = new List <long>();
List <int> blockLengths = new List <int>();
using (var stream = File.OpenRead(source))
{
long blockSize = 200 * 1024 * 1024; // 200M
if (lineBoundary)
{
byte[] Buffer = new byte[blockSize];
const byte EndOfLine = (byte)'\n';
long pos = 0;
int numRead = -1;
while ((numRead = stream.Read(Buffer, 0, Buffer.Length)) > 0)
{
int offset = -1;
for (int i = numRead - 1; i >= 0; i--)
{
if (Buffer[i].Equals(EndOfLine))
{
offset = i + 1;
break;
}
}
if (offset < 0) // the last line
{
blockOffsets.Add(pos);
blockLengths.Add(numRead);
}
else
{
blockOffsets.Add(pos);
blockLengths.Add(offset);
pos += offset;
stream.Seek(pos, SeekOrigin.Begin);
}
}
Buffer = null;
}
else
{
long pos;
for (pos = 0; pos < stream.Length; pos += blockSize)
{
blockOffsets.Add(pos);
blockLengths.Add((int)Math.Min(blockSize, stream.Length - pos));
}
}
}
long[] offsets = blockOffsets.ToArray();
int[] lengths = blockLengths.ToArray();
Debug.Assert(offsets.Length == lengths.Length);
// upload blocks
string guid = Guid.NewGuid().ToString();
string incomplete = target + guid;
List <string> partials = new List <string>();
int n = offsets.Length;
int retryNum = 0;
do
{
if (retryNum++ >= 3)
{
throw new Exception("Failed to upload " + source);
}
partials.Clear();
if (VC.StreamExists(incomplete))
{
VC.Delete(incomplete);
}
for (int i = 0; i < n; i++)
{
long offset = offsets[i];
int length = lengths[i];
string partial = string.Format("{0}_parts/{1}_{2}", target, guid, i);
int number = 0;
do
{
number++;
try
{
if (VC.StreamExists(partial))
{
if (VC.GetStreamInfo(partial, false).CommittedLength == length)
{
break;
}
VC.Delete(partial);
}
if (writeLog != null && i % 10 == 0)
{
writeLog(AdsAnswer.Logging.EventType.Information, string.Format("Uploading block {0}/{1}", i, n));
}
// Console.WriteLine(partial);
VC.Upload(source, partial, offset, length, true, new TimeSpan(DEFAULT_STREAM_EXPIRE_DAYS, 0, 0, 0), true);
VC.SealStream(partial);
}
catch (Exception ex)
{
AdsAnswer.Logging.Logger.Error("CosmosStream", "UploadLargeStream", "", ex.ToString());
if (number >= 3)
{
throw ex;
}
}
} while (!VC.StreamExists(partial) || VC.GetStreamInfo(partial, false).CommittedLength != length);
VC.Concatenate(partial, incomplete);
partials.Add(partial);
}
VC.SealStream(incomplete);
} while (!VC.StreamExists(incomplete) || VC.GetStreamInfo(incomplete, false).CommittedLength != fileSize);
// commit & clean up
if (VC.StreamExists(target))
{
VC.Delete(target);
}
VC.Concatenate(incomplete, target);
VC.Delete(incomplete);
foreach (var p in partials)
{
VC.Delete(p);
}
}