Beispiel #1
0
        /// <summary>
        /// Send an HTTP GET request to the specified URI and intepret the result as TResult.
        /// </summary>
        /// <typeparam name="TResult">The .NET class to use for response deserialization.</typeparam>
        /// <param name="uri">A uniform resource identifier that is the target of this GET request.</param>
        /// <param name="dataFormat">One of the data-interchange formats that Socrata supports. The default is JSON.</param>
        /// <returns>The HTTP response, deserialized into an object of type <typeparamref name="TResult"/>.</returns>
        internal TResult read <TResult>(Uri uri, SodaDataFormat dataFormat = SodaDataFormat.JSON)
            where TResult : class
        {
            var request = new SodaRequest(uri, "GET", AppToken, Username, password, dataFormat, null, RequestTimeout);

            return(request.ParseResponse <TResult>());
        }
        /// <summary>
        /// Export the error rows (if present).
        /// </summary>
        /// <param name="filepath">The output file (csv)</param>
        /// <param name="output">The specified transformed output</param>
        public void ExportErrorRows(string filepath, AppliedTransform output)
        {
            if (String.IsNullOrEmpty(filepath))
            {
                throw new ArgumentException("Filepath must be specified.", "filepath");
            }

            if (output == null)
            {
                throw new ArgumentException("Applied Transform required.", "output");
            }

            if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
            {
                throw new InvalidOperationException("Write operations require an authenticated client.");
            }

            var endpoint     = output.GetErrorRowEndpoint().Replace("{input_schema_id}", output.GetInputSchemaId()).Replace("{output_schema_id}", output.GetOutputSchemaId());
            var errorRowsUri = SodaUri.ForErrorRows(Host, endpoint);

            Debug.WriteLine(errorRowsUri.ToString());
            var downloadRowsRequest = new SodaRequest(errorRowsUri, "GET", null, username, password, SodaDataFormat.CSV, "");
            var result = downloadRowsRequest.ParseResponse <String>();

            System.IO.File.WriteAllText(filepath, result);
        }
        /// <summary>
        /// Await the completion of the update, optionally output the status.
        /// </summary>
        /// <param name="lambda">A lambda function for outputting status if desired.</param>
        public void AwaitCompletion(Action <string> lambda)
        {
            string status = "";
            Result r      = null;

            while (status != "successful" && status != "failure")
            {
                var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, username, password, SodaDataFormat.JSON);
                try
                {
                    r = jobRequest.ParseResponse <Result>();
                }
                catch (WebException webException)
                {
                    string message = webException.UnwrapExceptionMessage();
                    r = new Result()
                    {
                        Message = webException.Message, IsError = true, ErrorCode = message
                    };
                }
                status = r.Resource["task_sets"][0]["status"];
                lambda(status);
                System.Threading.Thread.Sleep(1000);
            }
        }
Beispiel #4
0
        /// <summary>
        /// Send an HTTP request of the specified method and interpret the result.
        /// </summary>
        /// <typeparam name="TPayload">The .NET class that represents the request payload.</typeparam>
        /// <typeparam name="TResult">The .NET class to use for response deserialization.</typeparam>
        /// <param name="uri">A uniform resource identifier that is the target of this GET request.</param>
        /// <param name="method">One of POST, PUT, or DELETE</param>
        /// <param name="payload">An object graph to serialize and send with the request.</param>
        /// <returns>The HTTP response, deserialized into an object of type <typeparamref name="TResult"/>.</returns>
        internal TResult write <TPayload, TResult>(Uri uri, string method, TPayload payload)
            where TPayload : class
            where TResult : class
        {
            var request = new SodaRequest(uri, method, AppToken, Username, password, SodaDataFormat.JSON, payload.ToJsonString(), RequestTimeout);

            return(request.ParseResponse <TResult>());
        }
        /// <summary>
        /// Get the specified source data.
        /// </summary>
        /// <param name="source">The result of the Source creation</param>
        /// <returns>A <see cref="Source"/>The updated Source object</returns>
        public Source GetSource(Source source)
        {
            if (source == null)
            {
                throw new ArgumentException("Source required.", "source");
            }
            var    sourceUri            = SodaUri.ForSource(Host, source.Self());
            var    sourceUpdateResponse = new SodaRequest(sourceUri, "GET", null, username, password, SodaDataFormat.JSON, "");
            Result result = sourceUpdateResponse.ParseResponse <Result>();

            return(new Source(result));
        }
Beispiel #6
0
        /// <summary>
        /// Replace any existing rows with the payload data, using the specified resource identifier.
        /// </summary>
        /// <param name="payload">A string of serialized data.</param>
        /// <param name="dataFormat">One of the data-interchange formats that Socrata supports, into which the payload has been serialized.</param>
        /// <param name="resourceId">The identifier (4x4) for a resource on the Socrata host to target.</param>
        /// <returns>A <see cref="SodaResult"/> indicating success or failure.</returns>
        /// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="dataFormat"/> is equal to <see cref="SodaDataFormat.XML"/>.</exception>
        /// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="resourceId"/> does not match the Socrata 4x4 pattern.</exception>
        /// <exception cref="System.InvalidOperationException">Thrown if this SodaClient was initialized without authentication credentials.</exception>
        public SodaResult Replace(string payload, SodaDataFormat dataFormat, string resourceId)
        {
            if (dataFormat == SodaDataFormat.XML)
            {
                throw new ArgumentOutOfRangeException("dataFormat", "XML is not a valid format for write operations.");
            }

            if (FourByFour.IsNotValid(resourceId))
            {
                throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier.");
            }

            if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password))
            {
                throw new InvalidOperationException("Write operations require an authenticated client.");
            }

            var uri = SodaUri.ForResourceAPI(Host, resourceId);

            var        request = new SodaRequest(uri, "PUT", AppToken, Username, password, dataFormat, payload);
            SodaResult result  = null;

            try
            {
                if (dataFormat == SodaDataFormat.JSON)
                {
                    result = request.ParseResponse <SodaResult>();
                }
                else if (dataFormat == SodaDataFormat.CSV)
                {
                    string resultJson = request.ParseResponse <string>();
                    result = Newtonsoft.Json.JsonConvert.DeserializeObject <SodaResult>(resultJson);
                }
            }
            catch (WebException webException)
            {
                string message = webException.UnwrapExceptionMessage();
                result = new SodaResult()
                {
                    Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
                };
            }
            catch (Exception ex)
            {
                result = new SodaResult()
                {
                    Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
                };
            }

            return(result);
        }
        /// <summary>
        /// Replace any existing rows with the payload data, using the specified resource identifier.
        /// </summary>
        /// <param name="method">One of update, replace, or delete</param>
        /// <param name="resourceId">The identifier (4x4) for a resource on the Socrata host to target.</param>
        /// <param name="permission">The permission level of the dataset, can be one of either "public" or "private".</param>
        /// <returns>A <see cref="Revision"/> newly created Revision.</returns>
        /// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="resourceId"/> does not match the Socrata 4x4 pattern.</exception>
        public Revision CreateRevision(string method, string resourceId, string permission = "private")
        {
            if (String.IsNullOrEmpty(method))
            {
                throw new ArgumentException("Method must be specified either update, replace, or delete.", "method");
            }

            if (FourByFour.IsNotValid(resourceId))
            {
                throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier.");
            }

            if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
            {
                throw new InvalidOperationException("Write operations require an authenticated client.");
            }

            var revisionUri = SodaUri.ForRevision(Host, resourceId);

            // Construct Revision Request body
            Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject();
            Newtonsoft.Json.Linq.JObject action  = new Newtonsoft.Json.Linq.JObject();
            action["type"]       = method;
            action["permission"] = permission;
            payload["action"]    = action;

            var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString());

            Result result = null;

            try
            {
                result = request.ParseResponse <Result>();
            }
            catch (WebException webException)
            {
                string message = webException.UnwrapExceptionMessage();
                result = new Result()
                {
                    Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
                };
            }
            catch (Exception ex)
            {
                result = new Result()
                {
                    Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
                };
            }
            return(new Revision(result));
        }
        /// <summary>
        /// Create a new dataset with a given name and permission level.
        /// </summary>
        /// <param name="name">The dataset name</param>
        /// <param name="permission">The permission level of the dataset, can be one of either "public" or "private".</param>
        /// <returns>A <see cref="Revision"/> newly created Revision.</returns>
        public Revision CreateDataset(string name, string permission = "private")
        {
            if (String.IsNullOrEmpty(name))
            {
                throw new ArgumentException("Dataset name required.", "name");
            }

            if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
            {
                throw new InvalidOperationException("Write operations require an authenticated client.");
            }

            var revisionUri = SodaUri.ForRevision(Host);

            // Construct Revision Request body
            Newtonsoft.Json.Linq.JObject payload  = new Newtonsoft.Json.Linq.JObject();
            Newtonsoft.Json.Linq.JObject metadata = new Newtonsoft.Json.Linq.JObject();
            Newtonsoft.Json.Linq.JObject action   = new Newtonsoft.Json.Linq.JObject();
            metadata["name"]     = name;
            action["type"]       = "replace";
            action["permission"] = permission;
            payload["action"]    = action;
            payload["metadata"]  = metadata;

            var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString());

            Result result = null;

            try
            {
                result = request.ParseResponse <Result>();
            }
            catch (WebException webException)
            {
                string message = webException.UnwrapExceptionMessage();
                result = new Result()
                {
                    Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
                };
            }
            catch (Exception ex)
            {
                result = new Result()
                {
                    Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
                };
            }
            return(new Revision(result));
        }
        /// <summary>
        /// Creates the source for the specified revision.
        /// </summary>
        /// <param name="data">A string of serialized data.</param>
        /// <param name="revision">The revision created as part of a create revision step.</param>
        /// <param name="dataFormat">The format of the data.</param>
        /// <param name="filename">The filename that should be associated with this upload.</param>
        /// <returns>A <see cref="Source"/> indicating success or failure.</returns>
        /// <exception cref="System.InvalidOperationException">Thrown if this SodaDSMAPIClient was initialized without authentication credentials.</exception>
        public Source CreateSource(string data, Revision revision, SodaDataFormat dataFormat = SodaDataFormat.CSV, string filename = "NewUpload")
        {
            if (String.IsNullOrEmpty(data))
            {
                throw new ArgumentException("Data must be provided.", "data");
            }

            if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
            {
                throw new InvalidOperationException("Write operations require an authenticated client.");
            }

            if (revision == null)
            {
                throw new ArgumentException("Revision required.", "revision");
            }

            var sourceUri = SodaUri.ForSource(Host, revision.GetSourceEndpoint());

            Debug.WriteLine(sourceUri.ToString());
            revision.GetRevisionNumber();

            // Construct Revision Request body
            Newtonsoft.Json.Linq.JObject payload      = new Newtonsoft.Json.Linq.JObject();
            Newtonsoft.Json.Linq.JObject source_type  = new Newtonsoft.Json.Linq.JObject();
            Newtonsoft.Json.Linq.JObject parse_option = new Newtonsoft.Json.Linq.JObject();
            source_type["type"]          = "upload";
            source_type["filename"]      = filename;
            parse_option["parse_source"] = true;
            payload["source_type"]       = source_type;
            payload["parse_options"]     = parse_option;

            var    createSourceRequest = new SodaRequest(sourceUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString());
            Result sourceOutput        = createSourceRequest.ParseResponse <Result>();
            string uploadDataPath      = sourceOutput.Links["bytes"];
            var    uploadUri           = SodaUri.ForUpload(Host, uploadDataPath);

            Debug.WriteLine(uploadUri.ToString());
            var fileUploadRequest = new SodaRequest(uploadUri, "POST", null, username, password, dataFormat, data);

            fileUploadRequest.SetDataType(SodaDataFormat.JSON);
            Result result = fileUploadRequest.ParseResponse <Result>();

            return(new Source(result));
        }
Beispiel #10
0
        /// <summary>
        /// Apply the source, transforms, and update to the specified dataset.
        /// </summary>
        /// <param name="outputSchema">A string of serialized data.</param>
        /// <param name="revision">A string of serialized data.</param>
        /// <returns>A <see cref="PipelineJob"/> for determining success or failure.</returns>
        public PipelineJob Apply(AppliedTransform outputSchema, Revision revision)
        {
            if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password))
            {
                throw new InvalidOperationException("Write operations require an authenticated client.");
            }

            if (outputSchema == null || revision == null)
            {
                throw new InvalidOperationException("Both the output schema and revision are required.");
            }

            Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject();
            payload["output_schema_id"] = outputSchema.GetOutputSchemaId();

            var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint());

            Debug.WriteLine(uri.ToString());
            var    applyRequest = new SodaRequest(uri, "PUT", null, username, password, SodaDataFormat.JSON, payload.ToString());
            Result result       = null;

            try
            {
                result = applyRequest.ParseResponse <Result>();
            }
            catch (WebException webException)
            {
                string message = webException.UnwrapExceptionMessage();
                result = new Result()
                {
                    Message = webException.Message, IsError = true, ErrorCode = message, Data = payload
                };
            }
            catch (Exception ex)
            {
                result = new Result()
                {
                    Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload
                };
            }

            return(new PipelineJob(SodaUri.ForJob(Host, revision.getRevisionLink()), username, password));
        }
Beispiel #11
0
        /// <summary>
        /// Delete a single row using the specified row identifier and the specified resource identifier.
        /// </summary>
        /// <param name="rowId">The identifier of the row to be deleted.</param>
        /// <param name="resourceId">The identifier (4x4) for a resource on the Socrata host to target.</param>
        /// <returns>A <see cref="SodaResult"/> indicating success or failure.</returns>
        /// <exception cref="System.ArgumentException">Thrown if the specified <paramref name="rowId"/> is null or empty.</exception>
        /// <exception cref="System.ArgumentOutOfRangeException">Thrown if the specified <paramref name="resourceId"/> does not match the Socrata 4x4 pattern.</exception>
        /// <exception cref="System.InvalidOperationException">Thrown if this SodaClient was initialized without authentication credentials.</exception>
        public SodaResult DeleteRow(string rowId, string resourceId)
        {
            if (String.IsNullOrEmpty(rowId))
            {
                throw new ArgumentException("Must specify the row to be deleted using its row identifier.", "rowId");
            }

            if (FourByFour.IsNotValid(resourceId))
            {
                throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier.");
            }

            if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password))
            {
                throw new InvalidOperationException("Write operations require an authenticated client.");
            }

            var uri = SodaUri.ForResourceAPI(Host, resourceId, rowId);

            var request = new SodaRequest(uri, "DELETE", AppToken, Username, password);

            return(request.ParseResponse <SodaResult>());
        }