public static void UploadToDfs(Uri dfsDirectory, string localPath) { DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsDirectory.AbsoluteUri); try { if (dfsDirectory.Scheme == "hdfs") { using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory)) { string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath); hdfs.UploadAll(localPath, dfsPath); } } else if (dfsDirectory.Scheme == "azureblob") { string account, key, container, blob; Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container); string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath); Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri); azure.PutDfsFile(dfsUri, localPath); } else if (dfsDirectory.Scheme == "file") { string dstPath = Path.Combine(dfsDirectory.AbsolutePath, Path.GetFileName(localPath)); File.Copy(localPath, dstPath); } } catch (Exception e) { DryadLogger.LogWarning("Failed to upload query plan: " + e.ToString()); } }
public static void UploadToDfs(Uri dfsDirectory, string dfsName, string payload) { byte[] payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload); DryadLogger.LogInformation("Uploading payload to " + dfsName + " at " + dfsDirectory.AbsoluteUri); try { if (dfsDirectory.Scheme == "hdfs") { using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory)) { string dfsPath = dfsDirectory.AbsolutePath + dfsName; hdfs.WriteAll(dfsPath, payloadBytes); } } else if (dfsDirectory.Scheme == "azureblob") { string account, key, container, blob; Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container); string dfsPath = blob + dfsName; Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key); azure.PutDfsFile(dfsUri, payloadBytes); } else if (dfsDirectory.Scheme == "file") { string dstPath = Path.Combine(dfsDirectory.AbsolutePath, dfsName); File.WriteAllBytes(dstPath, payloadBytes); } } catch (Exception e) { DryadLogger.LogWarning("Failed to upload final output: " + e.ToString()); } }