Implementing multipart file upload with extra params

Question

I am trying to replicate the following command:

curl -X POST --header 'Content-Type: multipart/form-data' --header 'Accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/1.7.0"' -F wikitext=%27%27%27Mahikari%27%27%27%20is%20a%20%5B%5BJapan%5D%5Dese%20%5B%5Bnew%20religious%20movement%5D%5D -F body_only=true -F   'https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html'

The file is passed as a url quoted parameter to curl.

The content of the original file is given as (with no trailing returns):

'''Mahikari''' is a [[Japan]]ese [[new religious movement]]

The only parameter I added, for now, is body_only=true

The expected and correct answer is:

<p id="mwAQ"><b id="mwAg">Mahikari</b> is a <a rel="mw:WikiLink" href="./Japan" title="Japan" id="mwAw">Japanese</a> <a rel="mw:WikiLink" href="./New_religious_movement" title="New religious movement" id="mwBA">new religious movement</a></p>

The code below is not returning anything (not even an error!):

package main

import (
    "bytes"
    "fmt"
    "io"
    // "io/ioutil"
    "log"
    "mime/multipart"
    "net/http"
    "os"
    "path/filepath"
)

// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
    file, err := os.Open(path)
    if err != nil {
        return nil, err
    }
    defer file.Close()

    // fileContents, err := ioutil.ReadAll(file)
    // if err != nil {
    //  return nil, err
    // }

    fi, err := file.Stat()
    if err != nil {
        return nil, err
    }

    body := new(bytes.Buffer)
    writer := multipart.NewWriter(body)
    part, err := writer.CreateFormFile(paramName, fi.Name())
    if err != nil {
        return nil, err
    }
    // part.Write(fileContents)
    io.Copy(part, file)

    for key, val := range params {
        _ = writer.WriteField(key, val)
    }
    err = writer.Close()
    if err != nil {
        return nil, err
    }

    request, err := http.NewRequest("POST", uri, body)
    request.Header.Add("Content-Type", writer.FormDataContentType())
    request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
    return request, err
}

func transformWikitextToHtml(path string) {
    extraParams := map[string]string{
        "body_only":       "true",
    }
    request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams, "file", path)
    if err != nil {
        log.Fatal(err)
    }
    client := &http.Client{}
    resp, err := client.Do(request)
    if err != nil {
        log.Fatal(err)
    } else {
        var bodyContent []byte
        fmt.Println(resp.StatusCode)
        fmt.Println(resp.Header)
        resp.Body.Read(bodyContent)
        resp.Body.Close()
        fmt.Println(bodyContent)
    }
}

func main() {
    transformWikitextToHtml("/tmp/2239217")
}

I set up the headers according to the documentation and what is expected. I tried a few things, as reading the file at once (commented out), but that didnt help. What am I missing?

...and when you say the code isn't returning anything, do you mean that not even the status code and header are being printed? — mkopriva
– mkopriva, Commented Jun 12, 2018 at 19:05
...also io.Copy and writer.WriteField too, return errors. Check every error every time. And you NewRequest result handling is bad practice, you are assuming that an *http.Request is always return even if you get an error, it's not, and so your calls to Header.Add would panic in case NewRequest returns an error. — mkopriva
– mkopriva, Commented Jun 12, 2018 at 19:11
No, I meant, it's returning a 400 which is obviously the problem, but because I emulated everything I could think of from curl cmd, I am not sure what I am missing here. — John Difool
– John Difool, Commented Jun 12, 2018 at 19:11

eugenioy · Accepted Answer · 2018-06-12 19:40:07Z

In your CURL request, you are sending wikitext as a field (-F wikitext=...).

However, in your code you are sending it as a file part.

If you send that as a field it will work as you expect.

Just include the file contents as an additional extra field in your code:

func transformWikitextToHtml(path string) {
    fileBytes, err := ioutil.ReadFile(path)
    if err != nil {
        log.Fatal(err)
    }
    extraParams := map[string]string{
        "body_only":       "true",
        "wikitext": string(fileBytes),
    }
    // rest of the code should be as you posted
}

Then of course, remove the parts of newfileUploadRequest that work with the path and file param name, which are not needed any more.

Also, when writing the response body, you had a small bug and it was not printing anything even once the code was fixed, so please replace that part with:

    bodyBytes, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println(string(bodyBytes))

Full working code:

package main

import (
    "bytes"
    "fmt"
    "log"
    "mime/multipart"
    "net/http"
    "io/ioutil"
)

// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string) (*http.Request, error) {

    body := new(bytes.Buffer)
    writer := multipart.NewWriter(body)

    for key, val := range params {
        err  := writer.WriteField(key, val)
        if err != nil {
            log.Fatal(err)
        }
    }
    err := writer.Close()
    if err != nil {
        return nil, err
    }

    request, err := http.NewRequest("POST", uri, body)
    request.Header.Add("Content-Type", writer.FormDataContentType())
    request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
    return request, err
}

func transformWikitextToHtml(path string) {
    fileBytes, err := ioutil.ReadFile(path)
    if err != nil {
        log.Fatal(err)
    }
    extraParams := map[string]string{
        "body_only":       "true",
        "wikitext": string(fileBytes),
    }
    request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams)
    if err != nil {
        log.Fatal(err)
    }
    client := &http.Client{}
    resp, err := client.Do(request)
    if err != nil {
        log.Fatal(err)
    } else {
        fmt.Println(resp.StatusCode)
        fmt.Println(resp.Header)
        bodyBytes, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            log.Fatal(err)
        }
        fmt.Println(string(bodyBytes))
    }
}

func main() {
    transformWikitextToHtml("/tmp/2239217")
}

Thanks for taking the time to edit the solution. I was trying to make curl use a file param and it stubbornly declined. Looking at your sample, I noticed that you use the byte array converted to string directly wihout path-escaping the content since it's just stuffed in the body of the request (vs passed as a cmd line arg?).

Collectives™ on Stack Overflow

Implementing multipart file upload with extra params

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related