public void Copy(CloudFile source, CloudFile target)
{
using (var stream = Read(source, null))
{
Write(target, stream, null);
}
}
public static void FormatProperties(CloudFile cloudFile)
{
StringBuilder sb = new StringBuilder();
sb.Append("{$v=1");
if (cloudFile.IsHistory)
{
sb.Append(";hist");
}
sb.Append(";u=");
var seconds = (int)((cloudFile.Timestamp - BaseTime).TotalSeconds);
sb.Append(seconds);
if (!string.IsNullOrEmpty(cloudFile.Fingerprint))
{
sb.Append(";fp=");
sb.Append(cloudFile.Fingerprint);
}
if (cloudFile.Format== CloudFileFormat.SevenZip)
{
sb.Append(";7z");
}
sb.Append("$}");
var filename = Path.GetFileNameWithoutExtension(cloudFile.Name) +
"." + sb.ToString() +
Path.GetExtension(cloudFile.Name);
cloudFile.UniqueName = filename;
}
public void Delete(CloudFile cloudFile)
{
string filePath = Path.Combine(this.rootPath, cloudFile.FullName());
if (File.Exists(filePath))
{
File.Delete(filePath);
}
}
public static void ParseProperties(CloudFile cloudFile)
{
var filename = Path.GetFileName(cloudFile.UniqueName);
var propertyMatch = ProtertyPatten.Match(filename);
if (propertyMatch.Success)
{
var property = propertyMatch.Groups[1].Value;
cloudFile.Name = ProtertyPatten.Replace(filename, string.Empty, 1);
cloudFile.Format = CloudFileFormat.Original;
foreach (var p in property.Split(';'))
{
if (string.IsNullOrEmpty(p))
{
continue;
}
if (string.Equals(p, "7z", StringComparison.OrdinalIgnoreCase))
{
cloudFile.Format = CloudFileFormat.SevenZip;
continue;
}
if (string.Equals(p, "hist", StringComparison.OrdinalIgnoreCase))
{
cloudFile.IsHistory = true;
continue;
}
var pare = p.Split('=');
if (pare.Length == 2)
{
string key = pare[0].Trim();
string value = pare[1].Trim();
switch (key)
{
case "u":
var seconds = (int)decimal.Parse(value);
cloudFile.Timestamp = BaseTime.AddSeconds(seconds);
break;
case "m":
case "fp":
cloudFile.Fingerprint = value;
break;
default:
continue;
}
}
}
}
}
public override Stream Read(CloudFile cloudFile, OnProgress onProgress)
{
if (onProgress != null)
{
onProgress(cloudFile, ProcessTypes.Download, 0, cloudFile.Size);
}
var result = connector.Read(cloudFile, (pos, size) => {
if (onProgress != null)
{
onProgress(cloudFile, ProcessTypes.Download, pos, size);
}
});
if (onProgress != null)
{
onProgress(cloudFile, ProcessTypes.Download, cloudFile.Size, cloudFile.Size);
}
return result;
}
public Stream Read(CloudFile cloudFile, OnProgress onProgress)
{
if (cloudFile.Format == CloudFileFormat.Original)
{
return provider.Read(cloudFile, onProgress);
}
else
{
Stream stream = provider.Read(cloudFile, onProgress);
using (var decoder = new SevenZip.SevenZipExtractor(stream, password))
{
if (!decoder.Check())
{
return stream;
}
else
{
Stream result = new MemoryStream();
if (decoder.FilesCount != 1)
{
throw new SevenZipProviderException("Cannot extract 7z package which contains no file or more than 1 file.");
}
decoder.Extracting += new EventHandler<ProgressEventArgs>(
(o, p) =>
{
if (onProgress != null)
{
onProgress(cloudFile, ProcessTypes.Decompress, (p.PercentDone * stream.Length), stream.Length);
}
});
decoder.ExtractFile(0, result);
stream.Close();
result.Position = 0;
return result;
}
}
}
}
public void Write(CloudFile cloudFile, Stream stream, Action callback = null)
{
if (cloudFile.Size <= Baidu.TempFileSize)
{
WriteSingleFile(cloudFile, stream, callback);
}
else
{
WriteSuperFile(cloudFile, stream, callback);
}
}
public Stream Read(CloudFile cloudFile, Action<long, long> callback)
{
var path = FormatPath(cloudFile.FullUniqueName());
string url = string.Format(Baidu.PcsDownloadUrl, token, path);
long finished = 0;
DateTime lastLog = DateTime.MinValue;
var httpResponse = new HttpWebResponseWrapper(HttpHelper.HttpGet(url),
p =>
{
if (callback != null)
{
callback(p, cloudFile.CloudSize);
}
finished += p;
if ((DateTime.Now - lastLog).TotalMilliseconds > 3000 || finished >= cloudFile.CloudSize)
{
logger.InfoFormat("Baidu reading [{0}], [{1}% of {2}MB]",
cloudFile.FullUniqueName(),
Math.Round((decimal)finished * 100 / cloudFile.CloudSize),
Math.Round((decimal)cloudFile.Size / 1024 / 1024, 2)
);
lastLog = DateTime.Now;
}
});
string tempFileName = Path.GetTempFileName();
var result = File.Create(tempFileName, 1024 * 4, FileOptions.DeleteOnClose);
httpResponse.CopyTo(result);
return result;
}
public abstract void Delete(CloudFile cloudFile);
public abstract void Copy(CloudFile source, CloudFile target);
public void Copy(CloudFile source, CloudFile target)
{
provider.Copy(source, target);
}
public void Delete(CloudFile cloudFile)
{
provider.Delete(cloudFile);
}
public override void Copy(CloudFile source, CloudFile target)
{
EnsureFolder(target.FullDirectory());
connector.Copy(source.FullUniqueName(), target.FullUniqueName());
}
private void WriteSuperFile(CloudFile cloudFile, Stream stream, Action callback = null)
{
string url = string.Format(Baidu.PcsUploadTmpUrl, token);
var fileSize = stream.Length - stream.Position;
var tempFileSize = Math.Max(fileSize / 1024, Baidu.TempFileSize);
int blockCount = (int)Math.Ceiling((double)(fileSize) / tempFileSize);
string[] blocks = new string[blockCount];
int errorCount = 0;
int successCount = 0;
while (true)
{
var i = GetNextBlockIndex(blocks);
if (i < 0)
break;
stream.Position = i * tempFileSize;
try
{
var response = HttpHelper.HttpUploadFileForString(url, stream, tempFileSize,
() =>
{
logger.InfoFormat("Writing baidu file [{0}], {1}% of {2}MB",
cloudFile.UniqueName,
Math.Round((double)stream.Position * 100 / stream.Length, 2),
Math.Round(((double)stream.Length / 1024 / 1024)), 2);
if (callback != null)
callback();
}
);
var tempFileInfo = Baidu.Serializer.Deserialize<BaiduTempFileInfo>(response);
blocks[i] = tempFileInfo.md5;
successCount++;
}
catch (Exception ex)
{
errorCount++;
int maxErrorCount = Math.Max(blockCount / 10, 3);
if ((errorCount>1 && errorCount > successCount) || errorCount > maxErrorCount)
{
throw ex;
}
}
}
CreateSuperFile(cloudFile, blocks);
}
private string CreateSuperFile(CloudFile cloudFile, string[] blocks)
{
string url = string.Format(Baidu.PcsCreateSuperfileUrl, token, FormatPath(cloudFile.FullUniqueName()));
StringBuilder json = new StringBuilder();
json.Append("{\"block_list\":[");
for (int i = 0; i < blocks.Length; i++)
{
if (i > 0)
{
json.Append(",");
}
json.AppendFormat("\"{0}\"", blocks[i]);
}
json.Append("]}");
var buffer = Encoding.ASCII.GetBytes("param=" + HttpUtility.UrlEncode(json.ToString()));
using (MemoryStream stream = new MemoryStream())
{
stream.Write(buffer, 0, buffer.Length);
stream.Position = 0;
return HttpHelper.HttpPostForString(url, stream);
}
}
public void Write(CloudFile cloudFile, Stream stream, OnProgress onProgress)
{
var path = Path.Combine(this.rootPath, cloudFile.FullName());
string directory = Path.GetDirectoryName(path);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
using (FileStream targetFileStream = new FileStream(path, FileMode.OpenOrCreate))
{
stream.CopyTo(targetFileStream);
}
File.SetLastWriteTime(path, cloudFile.Timestamp);
}
public Stream Read(CloudFile cloudFile, OnProgress onProgress)
{
string filePath = Path.Combine(this.rootPath, cloudFile.FullName().TrimStart('/'));
var result = new FileStream(filePath, FileMode.Open, FileAccess.Read);
cloudFile.Fingerprint = HashHelper.MD5(result);
result.Position = 0;
return result;
}
public override List<CloudItem> RealList(string path)
{
path = Path.Combine(this.rootPath, path);
if (Directory.Exists(path))
{
var fileSystemInfos = new DirectoryInfo(path).GetFileSystemInfos();
if (fileSystemInfos != null && fileSystemInfos.Length > 0)
{
var result = new List<CloudItem>();
foreach (var item in fileSystemInfos)
{
if ((item.Attributes & FileAttributes.Hidden) == FileAttributes.Hidden)
{
continue;
}
if (item is DirectoryInfo )
{
var cloudFolder = new CloudFolder();
cloudFolder.Name = Path.GetFileName(item.Name);
result.Add(cloudFolder);
}
else if (item is FileInfo)
{
var fileInfo = item as FileInfo;
if (fileInfo.Length > 0)
{
var cloudFile = new CloudFile();
cloudFile.UniqueName = item.Name;
cloudFile.CloudSize = fileInfo.Length;
result.Add(cloudFile);
}
}
}
return result;
}
}
return null;
}
public override Stream Read(CloudFile cloudFile, OnProgress onProgress)
{
string filePath = Path.Combine(this.rootPath, cloudFile.FullUniqueName());
return new FileStream(filePath, FileMode.Open, FileAccess.Read);
}
private void WriteSingleFile(CloudFile cloudFile, Stream stream, Action callback = null)
{
string url = string.Format(Baidu.PcsUploadUrl, token, FormatPath(cloudFile.FullUniqueName()));
var response = HttpHelper.HttpUploadFileForString(url, stream, 0,
() =>
{
logger.InfoFormat("Writing baidu file [{0}], {1}% of {2}MB",
cloudFile.Name,
Math.Round((double)stream.Position * 100 / stream.Length, 2),
Math.Round((double)(stream.Length / 1024 / 1024)));
if (callback != null)
callback();
}
);
}
public List<CloudItem> List(string path)
{
path = Path.Combine(this.rootPath, path.TrimStart('/'));
if (!Directory.Exists(path))
{
return null;
//throw new FileNotFoundException(string.Format("LocalHDProvider path does not exist, path = {0}", path));
}
var fileSystemInfos = new DirectoryInfo(path).GetFileSystemInfos();
if (fileSystemInfos != null && fileSystemInfos.Length > 0)
{
var result = new List<CloudItem>();
foreach (var item in fileSystemInfos)
{
if ((item.Attributes & FileAttributes.Hidden) == FileAttributes.Hidden)
{
continue;
}
if (item is DirectoryInfo)
{
var cloudFolder = new CloudFolder();
cloudFolder.Name = Path.GetFileName(item.Name);
result.Add(cloudFolder);
}
else if (item is FileInfo)
{
var fileInfo = item as FileInfo;
if (fileInfo.Length > 0)
{
var cloudFile = new CloudFile();
cloudFile.Name = fileInfo.Name;
cloudFile.Size = fileInfo.Length;
cloudFile.Timestamp = fileInfo.LastWriteTime;
result.Add(cloudFile);
}
}
}
return result;
}
return null;
}
protected override void RealWrite(CloudFile cloudFile, Stream stream, OnProgress onProgress)
{
if (onProgress != null)
{
onProgress(cloudFile, ProcessTypes.Upload, 0, stream.Length);
}
connector.Write(cloudFile, stream, () =>
{
if (onProgress != null)
{
onProgress(cloudFile, ProcessTypes.Upload, stream.Position, stream.Length);
}
});
if (onProgress != null)
{
onProgress(cloudFile, ProcessTypes.Upload, stream.Length, stream.Length);
}
}
public virtual void Move(CloudFile source, string newPath)
{
var path = source.FullName();
var oldFolder = Path.GetDirectoryName(path);
var newFolder = Path.GetDirectoryName(newPath);
}
public override void Delete(CloudFile cloudFile)
{
connector.Delete(cloudFile.FullUniqueName());
}
public abstract Stream Read(CloudFile cloudFile, OnProgress onProgress);
public override List<CloudItem> RealList(string path)
{
BaiduListResult listResult = null;
try
{
listResult = connector.List(path);
}
catch (WebException ex)
{
throw new CloudProviderOperationFailedException("Baidu list failed, path = " + path, ex);
}
if (listResult.list != null && listResult.list.Count > 0)
{
var result = new List<CloudItem>();
foreach (var item in listResult.list)
{
if (item.isdir == 1)
{
var cloudFolder = new CloudFolder();
cloudFolder.Name = Path.GetFileName(item.path);
result.Add(cloudFolder);
}
else
{
var cloudFile = new CloudFile();
cloudFile.UniqueName = item.path;
cloudFile.CloudSize = item.size;
result.Add(cloudFile);
}
}
return result;
}
return null;
}
public void Write(CloudFile cloudFile, Stream stream, OnProgress onProgress = null)
{
PropertyParser.FormatProperties(cloudFile);
var sameFile = cache.GetFileByFingerprint(cloudFile.Fingerprint);
if (sameFile == null && !string.IsNullOrEmpty(cloudFile.MD5))
{
sameFile = cache.GetFileByFingerprint(cloudFile.MD5);
}
if (sameFile != null)
{
logger.InfoFormat("Find same file, using copy instead of write. {0}", cloudFile.FullUniqueName());
Copy(sameFile, cloudFile);
}
else
{
RealWrite(cloudFile, stream, onProgress);
}
cache.AddItem(cloudFile);
}
protected abstract void RealWrite(CloudFile cloudFile, Stream stream, OnProgress onProgress);
public void Write(CloudFile cloudFile, Stream stream, OnProgress onProgress)
{
logger.DebugFormat("7zip, start writing [{0}]", cloudFile.FullName());
cloudFile.Format = CloudFileFormat.SevenZip;
//cloudFile.Fingerprint = HashHelper.MD5(cloudFile.Fingerprint + password);
stream.Position = 0;
// set the name into the package
Dictionary<String, Stream> dir = new Dictionary<string, Stream>();
dir[cloudFile.Name] = stream;
// drive low level provider to write data
using (MemoryStream compressedStream = new MemoryStream())
{
SevenZipCompressor compressor = new SevenZipCompressor();
compressor.Compressing += new EventHandler<ProgressEventArgs>(
(o, p) => {
onProgress(cloudFile, ProcessTypes.Compress, (p.PercentDone * stream.Length / 100), stream.Length);
logger.InfoFormat("Compressing [{0}], [{1}% of {2}MB]", cloudFile.Name, p.PercentDone, (int)(stream.Length / 1024 / 1024));
});
compressor.CompressStreamDictionary(dir, compressedStream, password);
compressedStream.Position = 0;
cloudFile.MD5 = cloudFile.Fingerprint;
cloudFile.Fingerprint = HashHelper.MD5(cloudFile.Fingerprint+password);
provider.Write(cloudFile, compressedStream, onProgress);
}
logger.DebugFormat("7zip, finish writing [{0}]", cloudFile.FullName());
}