Function fetchAndSubmitForm

scrape/forms.go:84–119

fetchAndSubmitForm will fetch the page at urlStr, then parse and submit the first form found. setValues will be called with the parsed form values, allowing the caller to set any custom form values. Form submission will always use the POST method, regardless of the value of the method attribute in t

(client *http.Client, urlStr string, setValues func(url.Values))

// form values. Form submission will always use the POST method, regardless of the value of the
// method attribute in the form.  The response from submitting the parsed form is returned.
func fetchAndSubmitForm(client *http.Client, urlStr string, setValues func(url.Values)) (*http.Response, error) {
	resp, err := client.Get(urlStr)
	if err != nil {
		return nil, fmt.Errorf("error fetching url %q: %v", urlStr, err)
	}

	defer resp.Body.Close()
	root, err := html.Parse(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error parsing response: %v", err)
	}

	forms := parseForms(root)
	if len(forms) == 0 {
		return nil, fmt.Errorf("no forms found at %q", urlStr)
	}
	form := forms[0]

	actionURL, err := url.Parse(form.Action)
	if err != nil {
		return nil, fmt.Errorf("error parsing form action URL %q: %v", form.Action, err)
	}
	actionURL = resp.Request.URL.ResolveReference(actionURL)

	// allow caller to fill out the form
	if setValues != nil {
		setValues(form.Values)
	}

	resp, err = client.PostForm(actionURL.String(), form.Values)
	if err != nil {
		return nil, fmt.Errorf("error posting form: %v", err)
	}

	return resp, nil
}