You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
starnet/curl.go

1451 lines
33 KiB
Go

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

package starnet
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
)
const (
HEADER_FORM_URLENCODE = `application/x-www-form-urlencoded`
HEADER_FORM_DATA = `multipart/form-data`
HEADER_JSON = `application/json`
HEADER_PLAIN = `text/plain`
)
var (
DefaultDialTimeout = 5 * time.Second
DefaultTimeout = 10 * time.Second
DefaultFetchRespBody = false
)
func UrlEncodeRaw(str string) string {
strs := strings.Replace(url.QueryEscape(str), "+", "%20", -1)
return strs
}
func UrlEncode(str string) string {
return url.QueryEscape(str)
}
func UrlDecode(str string) (string, error) {
return url.QueryUnescape(str)
}
func BuildQuery(queryData map[string]string) string {
query := url.Values{}
for k, v := range queryData {
query.Add(k, v)
}
return query.Encode()
}
// BuildPostForm takes a map of string keys and values, converts it into a URL-encoded query string,
// and then converts that string into a byte slice. This function is useful for preparing data for HTTP POST requests,
// where the server expects the request body to be URL-encoded form data.
//
// Parameters:
// queryMap: A map where the key-value pairs represent the form data to be sent in the HTTP POST request.
//
// Returns:
// A byte slice representing the URL-encoded form data.
func BuildPostForm(queryMap map[string]string) []byte {
return []byte(BuildQuery(queryMap))
}
func Get(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "GET", opts...).Do()
}
func Post(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "POST", opts...).Do()
}
func Options(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "OPTIONS", opts...).Do()
}
func Put(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "PUT", opts...).Do()
}
func Delete(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "DELETE", opts...).Do()
}
func Head(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "HEAD", opts...).Do()
}
func Patch(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "PATCH", opts...).Do()
}
func Trace(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "TRACE", opts...).Do()
}
func Connect(uri string, opts ...RequestOpt) (*Response, error) {
return NewSimpleRequest(uri, "CONNECT", opts...).Do()
}
type Request struct {
ctx context.Context
uri string
method string
errInfo error
RequestOpts
}
func (r *Request) Method() string {
return r.method
}
func (r *Request) SetMethod(method string) error {
method = strings.ToUpper(method)
if !validMethod(method) {
return fmt.Errorf("invalid method: %s", method)
}
r.method = method
r.rawRequest.Method = method
return nil
}
func (r *Request) SetMethodNoError(method string) *Request {
r.SetMethod(method)
return r
}
func (r *Request) Uri() string {
return r.uri
}
func (r *Request) SetUri(uri string) error {
u, err := url.Parse(uri)
if err != nil {
return fmt.Errorf("parse uri error: %s", err)
}
r.uri = uri
u.Host = removeEmptyPort(u.Host)
r.rawRequest.Host = u.Host
r.rawRequest.URL = u
return nil
}
func (r *Request) SetUriNoError(uri string) *Request {
r.SetUri(uri)
return r
}
func (r *Request) RawRequest() *http.Request {
return r.rawRequest
}
func (r *Request) SetRawRequest(rawRequest *http.Request) *Request {
r.rawRequest = rawRequest
return r
}
func (r *Request) RawClient() *http.Client {
return r.rawClient
}
func (r *Request) SetRawClient(rawClient *http.Client) *Request {
r.rawClient = rawClient
return r
}
func (r *Request) Do() (*Response, error) {
return Curl(r)
}
func (r *Request) Get() (*Response, error) {
err := r.SetMethod("GET")
if err != nil {
return nil, err
}
return Curl(r)
}
func (r *Request) Post(data []byte) (*Response, error) {
err := r.SetMethod("POST")
if err != nil {
return nil, err
}
r.bodyDataBytes = data
r.bodyDataReader = nil
return Curl(r)
}
type RequestOpts struct {
rawRequest *http.Request
rawClient *http.Client
alreadyApply bool
bodyDataBytes []byte
bodyDataReader io.Reader
bodyFormData map[string][]string
bodyFileData []RequestFile
//以上优先度为 bodyDataReader> bodyDataBytes > bodyFormData > bodyFileData
FileUploadRecallFn func(filename string, upPos int64, total int64)
proxy string
timeout time.Duration
dialTimeout time.Duration
headers http.Header
cookies []*http.Cookie
transport *http.Transport
queries map[string][]string
disableRedirect bool
//doRawRequest=true 不对request修改直接发送
doRawRequest bool
//doRawClient=true 不对http client修改直接发送
doRawClient bool
//doRawTransPort=true 不对http transport修改直接发送
doRawTransport bool
skipTLSVerify bool
tlsConfig *tls.Config
autoFetchRespBody bool
customIP []string
alreadySetLookUpIPfn bool
lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)
customDNS []string
basicAuth [2]string
autoCalcContentLength bool
}
func (r *Request) AutoCalcContentLength() bool {
return r.autoCalcContentLength
}
// SetAutoCalcContentLength sets whether to automatically calculate the Content-Length header based on the request body.
// WARN: If set to true, the Content-Length header will be set to the length of the request body, which may cause issues with chunked transfer encoding.
// also the memory usage will be higher
// Note that this function will not work if doRawRequest is true
func (r *Request) SetAutoCalcContentLength(autoCalcContentLength bool) *Request {
r.autoCalcContentLength = autoCalcContentLength
return r
}
// BasicAuth returns the username and password provided in the request's Authorization header.
func (r *RequestOpts) BasicAuth() (string, string) {
return r.basicAuth[0], r.basicAuth[1]
}
// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
// Note: If doRawRequest is true, this function will nolonger work
func (r *RequestOpts) SetBasicAuth(username, password string) *RequestOpts {
r.basicAuth = [2]string{username, password}
return r
}
func (r *Request) DoRawTransport() bool {
return r.doRawTransport
}
func (r *Request) SetDoRawTransport(doRawTransport bool) *Request {
r.doRawTransport = doRawTransport
return r
}
func (r *Request) CustomDNS() []string {
return r.customDNS
}
// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) SetCustomDNS(customDNS []string) error {
for _, v := range customDNS {
if net.ParseIP(v) == nil {
return fmt.Errorf("invalid custom dns: %s", v)
}
}
r.customDNS = customDNS
return nil
}
// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) SetCustomDNSNoError(customDNS []string) *Request {
r.SetCustomDNS(customDNS)
return r
}
// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) AddCustomDNS(customDNS []string) error {
for _, v := range customDNS {
if net.ParseIP(v) == nil {
return fmt.Errorf("invalid custom dns: %s", v)
}
}
r.customDNS = customDNS
return nil
}
// Note: if LookUpIPfn is set, this function will not be used
func (r *Request) AddCustomDNSNoError(customDNS []string) *Request {
r.AddCustomDNS(customDNS)
return r
}
func (r *Request) LookUpIPfn() func(ctx context.Context, host string) ([]net.IPAddr, error) {
return r.lookUpIPfn
}
func (r *Request) SetLookUpIPfn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) *Request {
if lookUpIPfn == nil {
r.alreadySetLookUpIPfn = false
r.lookUpIPfn = net.DefaultResolver.LookupIPAddr
return r
}
r.lookUpIPfn = lookUpIPfn
r.alreadySetLookUpIPfn = true
return r
}
func (r *Request) CustomHostIP() []string {
return r.customIP
}
func (r *Request) SetCustomHostIP(customIP []string) *Request {
r.customIP = customIP
return r
}
func (r *Request) AddCustomHostIP(customIP string) *Request {
r.customIP = append(r.customIP, customIP)
return r
}
func (r *Request) BodyDataBytes() []byte {
return r.bodyDataBytes
}
func (r *Request) SetBodyDataBytes(bodyDataBytes []byte) *Request {
r.bodyDataBytes = bodyDataBytes
return r
}
func (r *Request) BodyDataReader() io.Reader {
return r.bodyDataReader
}
func (r *Request) SetBodyDataReader(bodyDataReader io.Reader) *Request {
r.bodyDataReader = bodyDataReader
return r
}
func (r *Request) BodyFormData() map[string][]string {
return r.bodyFormData
}
func (r *Request) SetBodyFormData(bodyFormData map[string][]string) *Request {
r.bodyFormData = bodyFormData
return r
}
func (r *Request) SetFormData(bodyFormData map[string][]string) *Request {
return r.SetBodyFormData(bodyFormData)
}
func (r *Request) AddFormMapData(bodyFormData map[string]string) *Request {
for k, v := range bodyFormData {
r.bodyFormData[k] = append(r.bodyFormData[k], v)
}
return r
}
func (r *Request) AddFormData(k, v string) *Request {
r.bodyFormData[k] = append(r.bodyFormData[k], v)
return r
}
func (r *Request) BodyFileData() []RequestFile {
return r.bodyFileData
}
func (r *Request) SetBodyFileData(bodyFileData []RequestFile) *Request {
r.bodyFileData = bodyFileData
return r
}
func (r *Request) Proxy() string {
return r.proxy
}
func (r *Request) SetProxy(proxy string) *Request {
r.proxy = proxy
return r
}
func (r *Request) Timeout() time.Duration {
return r.timeout
}
func (r *Request) SetTimeout(timeout time.Duration) *Request {
r.timeout = timeout
return r
}
func (r *Request) DialTimeout() time.Duration {
return r.dialTimeout
}
func (r *Request) SetDialTimeout(dialTimeout time.Duration) *Request {
r.dialTimeout = dialTimeout
return r
}
func (r *Request) Headers() http.Header {
return r.headers
}
func (r *Request) SetHeaders(headers http.Header) *Request {
r.headers = headers
return r
}
func (r *Request) AddHeader(key, val string) *Request {
r.headers.Add(key, val)
return r
}
func (r *Request) SetHeader(key, val string) *Request {
r.headers.Set(key, val)
return r
}
func (r *Request) SetContentType(ct string) *Request {
r.headers.Set("Content-Type", ct)
return r
}
func (r *Request) SetUserAgent(ua string) *Request {
r.headers.Set("User-Agent", ua)
return r
}
func (r *Request) DeleteHeader(key string) *Request {
r.headers.Del(key)
return r
}
func (r *Request) Cookies() []*http.Cookie {
return r.cookies
}
func (r *Request) SetCookies(cookies []*http.Cookie) *Request {
r.cookies = cookies
return r
}
func (r *Request) Transport() *http.Transport {
return r.transport
}
func (r *Request) SetTransport(transport *http.Transport) *Request {
r.transport = transport
return r
}
func (r *Request) Queries() map[string][]string {
return r.queries
}
func (r *Request) SetQueries(queries map[string][]string) *Request {
r.queries = queries
return r
}
func (r *Request) AddQueries(queries map[string]string) *Request {
for k, v := range queries {
r.queries[k] = append(r.queries[k], v)
}
return r
}
func (r *Request) AddQuery(key, value string) *Request {
r.queries[key] = append(r.queries[key], value)
return r
}
func (r *Request) DelQueryKv(key, value string) *Request {
if _, ok := r.queries[key]; !ok {
return r
}
for i, v := range r.queries[key] {
if v == value {
r.queries[key] = append(r.queries[key][:i], r.queries[key][i+1:]...)
}
}
return r
}
func (r *Request) DelQuery(key string) *Request {
if _, ok := r.queries[key]; !ok {
return r
}
delete(r.queries, key)
return r
}
func (r *Request) DisableRedirect() bool {
return r.disableRedirect
}
func (r *Request) SetDisableRedirect(disableRedirect bool) *Request {
r.disableRedirect = disableRedirect
return r
}
func (r *Request) DoRawRequest() bool {
return r.doRawRequest
}
func (r *Request) SetDoRawRequest(doRawRequest bool) *Request {
r.doRawRequest = doRawRequest
return r
}
func (r *Request) DoRawClient() bool {
return r.doRawClient
}
func (r *Request) SetDoRawClient(doRawClient bool) *Request {
r.doRawClient = doRawClient
return r
}
func (r *RequestOpts) SkipTLSVerify() bool {
return r.skipTLSVerify
}
func (r *Request) SetSkipTLSVerify(skipTLSVerify bool) *Request {
r.skipTLSVerify = skipTLSVerify
return r
}
func (r *Request) TlsConfig() *tls.Config {
return r.tlsConfig
}
func (r *Request) SetTlsConfig(tlsConfig *tls.Config) *Request {
r.tlsConfig = tlsConfig
return r
}
func (r *Request) AutoFetchRespBody() bool {
return r.autoFetchRespBody
}
func (r *Request) SetAutoFetchRespBody(autoFetchRespBody bool) *Request {
r.autoFetchRespBody = autoFetchRespBody
return r
}
func (r *Request) ResetReqHeader() *Request {
r.headers = make(http.Header)
return r
}
func (r *Request) ResetReqCookies() *Request {
r.cookies = []*http.Cookie{}
return r
}
func (r *Request) AddSimpleCookie(key, value string) *Request {
r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: "/"})
return r
}
func (r *Request) AddCookie(key, value, path string) *Request {
r.cookies = append(r.cookies, &http.Cookie{Name: key, Value: value, Path: path})
return r
}
func (r *Request) AddFile(formName, filepath string) error {
f, err := os.Open(filepath)
if err != nil {
return err
}
stat, err := f.Stat()
if err != nil {
return err
}
r.bodyFileData = append(r.bodyFileData, RequestFile{
FormName: formName,
FileName: stat.Name(),
FileData: f,
FileSize: stat.Size(),
FileType: "application/octet-stream",
})
return nil
}
func (r *Request) AddFileWithName(formName, filepath, filename string) error {
f, err := os.Open(filepath)
if err != nil {
return err
}
stat, err := f.Stat()
if err != nil {
return err
}
r.bodyFileData = append(r.bodyFileData, RequestFile{
FormName: formName,
FileName: filename,
FileData: f,
FileSize: stat.Size(),
FileType: "application/octet-stream",
})
return nil
}
func (r *Request) AddFileWithType(formName, filepath, filetype string) error {
f, err := os.Open(filepath)
if err != nil {
return err
}
stat, err := f.Stat()
if err != nil {
return err
}
r.bodyFileData = append(r.bodyFileData, RequestFile{
FormName: formName,
FileName: stat.Name(),
FileData: f,
FileSize: stat.Size(),
FileType: filetype,
})
return nil
}
func (r *Request) AddFileWithNameAndType(formName, filepath, filename, filetype string) error {
f, err := os.Open(filepath)
if err != nil {
return err
}
stat, err := f.Stat()
if err != nil {
return err
}
r.bodyFileData = append(r.bodyFileData, RequestFile{
FormName: formName,
FileName: filename,
FileData: f,
FileSize: stat.Size(),
FileType: filetype,
})
return nil
}
func (r *Request) AddFileNoError(formName, filepath string) *Request {
r.AddFile(formName, filepath)
return r
}
func (r *Request) AddFileWithNameNoError(formName, filepath, filename string) *Request {
r.AddFileWithName(formName, filepath, filename)
return r
}
func (r *Request) AddFileWithTypeNoError(formName, filepath, filetype string) *Request {
r.AddFileWithType(formName, filepath, filetype)
return r
}
func (r *Request) AddFileWithNameAndTypeNoError(formName, filepath, filename, filetype string) *Request {
r.AddFileWithNameAndType(formName, filepath, filename, filetype)
return r
}
type RequestFile struct {
FormName string
FileName string
FileData io.Reader
FileSize int64
FileType string
}
type RequestOpt func(opt *RequestOpts) error
// if doRawTransport is true, this function will nolonger work
func WithDialTimeout(timeout time.Duration) RequestOpt {
return func(opt *RequestOpts) error {
opt.dialTimeout = timeout
return nil
}
}
// if doRawTransport is true, this function will nolonger work
func WithTimeout(timeout time.Duration) RequestOpt {
return func(opt *RequestOpts) error {
opt.timeout = timeout
return nil
}
}
// if doRawTransport is true, this function will nolonger work
func WithTlsConfig(tlscfg *tls.Config) RequestOpt {
return func(opt *RequestOpts) error {
opt.tlsConfig = tlscfg
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithHeader(key, val string) RequestOpt {
return func(opt *RequestOpts) error {
opt.headers.Set(key, val)
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithHeaderMap(header map[string]string) RequestOpt {
return func(opt *RequestOpts) error {
for key, val := range header {
opt.headers.Set(key, val)
}
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithReader(r io.Reader) RequestOpt {
return func(opt *RequestOpts) error {
opt.bodyDataReader = r
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithBytes(r []byte) RequestOpt {
return func(opt *RequestOpts) error {
opt.bodyDataBytes = r
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithFormData(data map[string][]string) RequestOpt {
return func(opt *RequestOpts) error {
opt.bodyFormData = data
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithFileDatas(data []RequestFile) RequestOpt {
return func(opt *RequestOpts) error {
opt.bodyFileData = data
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithFileData(data RequestFile) RequestOpt {
return func(opt *RequestOpts) error {
opt.bodyFileData = append(opt.bodyFileData, data)
return nil
}
}
// if doRawRequest is true, this function will nolonger work
func WithAddFile(formName, filepath string) RequestOpt {
return func(opt *RequestOpts) error {
f, err := os.Open(filepath)
if err != nil {
return err
}
stat, err := f.Stat()
if err != nil {
return err
}
opt.bodyFileData = append(opt.bodyFileData, RequestFile{
FormName: formName,
FileName: stat.Name(),
FileData: f,
FileSize: stat.Size(),
FileType: "application/octet-stream",
})
return nil
}
}
func WithAddFileWithName(formName, filepath, filename string) RequestOpt {
return func(opt *RequestOpts) error {
f, err := os.Open(filepath)
if err != nil {
return err
}
stat, err := f.Stat()
if err != nil {
return err
}
opt.bodyFileData = append(opt.bodyFileData, RequestFile{
FormName: formName,
FileName: filename,
FileData: f,
FileSize: stat.Size(),
FileType: "application/octet-stream",
})
return nil
}
}
func WithAddFileWithType(formName, filepath, filetype string) RequestOpt {
return func(opt *RequestOpts) error {
f, err := os.Open(filepath)
if err != nil {
return nil
}
stat, err := f.Stat()
if err != nil {
return nil
}
opt.bodyFileData = append(opt.bodyFileData, RequestFile{
FormName: formName,
FileName: stat.Name(),
FileData: f,
FileSize: stat.Size(),
FileType: filetype,
})
return nil
}
}
func WithAddFileWithNameAndType(formName, filepath, filename, filetype string) RequestOpt {
return func(opt *RequestOpts) error {
f, err := os.Open(filepath)
if err != nil {
return err
}
stat, err := f.Stat()
if err != nil {
return err
}
opt.bodyFileData = append(opt.bodyFileData, RequestFile{
FormName: formName,
FileName: filename,
FileData: f,
FileSize: stat.Size(),
FileType: filetype,
})
return nil
}
}
func WithFetchRespBody(fetch bool) RequestOpt {
return func(opt *RequestOpts) error {
opt.autoFetchRespBody = fetch
return nil
}
}
func WithCookies(ck []*http.Cookie) RequestOpt {
return func(opt *RequestOpts) error {
opt.cookies = ck
return nil
}
}
func WithCookie(key, val, path string) RequestOpt {
return func(opt *RequestOpts) error {
opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: path})
return nil
}
}
func WithSimpleCookie(key, val string) RequestOpt {
return func(opt *RequestOpts) error {
opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: "/"})
return nil
}
}
func WithCookieMap(header map[string]string, path string) RequestOpt {
return func(opt *RequestOpts) error {
for key, val := range header {
opt.cookies = append(opt.cookies, &http.Cookie{Name: key, Value: val, Path: path})
}
return nil
}
}
func WithQueries(queries map[string][]string) RequestOpt {
return func(opt *RequestOpts) error {
opt.queries = queries
return nil
}
}
func WithAddQueries(queries map[string]string) RequestOpt {
return func(opt *RequestOpts) error {
for k, v := range queries {
opt.queries[k] = append(opt.queries[k], v)
}
return nil
}
}
func WithAddQuery(key, val string) RequestOpt {
return func(opt *RequestOpts) error {
opt.queries[key] = append(opt.queries[key], val)
return nil
}
}
func WithProxy(proxy string) RequestOpt {
return func(opt *RequestOpts) error {
opt.proxy = proxy
return nil
}
}
func WithProcess(fn func(string, int64, int64)) RequestOpt {
return func(opt *RequestOpts) error {
opt.FileUploadRecallFn = fn
return nil
}
}
func WithContentType(ct string) RequestOpt {
return func(opt *RequestOpts) error {
opt.headers.Set("Content-Type", ct)
return nil
}
}
func WithUserAgent(ua string) RequestOpt {
return func(opt *RequestOpts) error {
opt.headers.Set("User-Agent", ua)
return nil
}
}
func WithSkipTLSVerify(skip bool) RequestOpt {
return func(opt *RequestOpts) error {
opt.skipTLSVerify = skip
return nil
}
}
func WithDisableRedirect(disable bool) RequestOpt {
return func(opt *RequestOpts) error {
opt.disableRedirect = disable
return nil
}
}
func WithDoRawRequest(doRawRequest bool) RequestOpt {
return func(opt *RequestOpts) error {
opt.doRawRequest = doRawRequest
return nil
}
}
func WithDoRawClient(doRawClient bool) RequestOpt {
return func(opt *RequestOpts) error {
opt.doRawClient = doRawClient
return nil
}
}
func WithDoRawTransport(doRawTrans bool) RequestOpt {
return func(opt *RequestOpts) error {
opt.doRawTransport = doRawTrans
return nil
}
}
func WithTransport(hs *http.Transport) RequestOpt {
return func(opt *RequestOpts) error {
opt.transport = hs
return nil
}
}
func WithRawRequest(req *http.Request) RequestOpt {
return func(opt *RequestOpts) error {
opt.rawRequest = req
return nil
}
}
func WithRawClient(hc *http.Client) RequestOpt {
return func(opt *RequestOpts) error {
opt.rawClient = hc
return nil
}
}
func WithCustomHostIP(ip []string) RequestOpt {
return func(opt *RequestOpts) error {
if len(ip) == 0 {
return nil
}
for _, v := range ip {
if net.ParseIP(v) == nil {
return fmt.Errorf("invalid custom ip: %s", v)
}
}
opt.customIP = ip
return nil
}
}
func WithAddCustomHostIP(ip string) RequestOpt {
return func(opt *RequestOpts) error {
if net.ParseIP(ip) == nil {
return fmt.Errorf("invalid custom ip: %s", ip)
}
opt.customIP = append(opt.customIP, ip)
return nil
}
}
func WithLookUpFn(lookUpIPfn func(ctx context.Context, host string) ([]net.IPAddr, error)) RequestOpt {
return func(opt *RequestOpts) error {
if lookUpIPfn == nil {
opt.alreadySetLookUpIPfn = false
opt.lookUpIPfn = net.DefaultResolver.LookupIPAddr
return nil
}
opt.lookUpIPfn = lookUpIPfn
opt.alreadySetLookUpIPfn = true
return nil
}
}
// WithCustomDNS will use custom dns to resolve the host
// Note: if LookUpIPfn is set, this function will not be used
func WithCustomDNS(customDNS []string) RequestOpt {
return func(opt *RequestOpts) error {
for _, v := range customDNS {
if net.ParseIP(v) == nil {
return fmt.Errorf("invalid custom dns: %s", v)
}
}
opt.customDNS = customDNS
return nil
}
}
// WithAddCustomDNS will use a custom dns to resolve the host
// Note: if LookUpIPfn is set, this function will not be used
func WithAddCustomDNS(customDNS string) RequestOpt {
return func(opt *RequestOpts) error {
if net.ParseIP(customDNS) == nil {
return fmt.Errorf("invalid custom dns: %s", customDNS)
}
opt.customDNS = append(opt.customDNS, customDNS)
return nil
}
}
// WithAutoCalcContentLength sets whether to automatically calculate the Content-Length header based on the request body.
// WARN: If set to true, the Content-Length header will be set to the length of the request body, which may cause issues with chunked transfer encoding.
// also the memory usage will be higher
// Note that this function will not work if doRawRequest is true
func (r *RequestOpts) WithAutoCalcContentLength(autoCalcContentLength bool) RequestOpt {
return func(opt *RequestOpts) error {
r.autoCalcContentLength = autoCalcContentLength
return nil
}
}
type Response struct {
*http.Response
req Request
data *Body
}
type Body struct {
full []byte
raw io.ReadCloser
isFull bool
}
func (b *Body) readAll() {
if !b.isFull {
b.full, _ = io.ReadAll(b.raw)
b.isFull = true
b.raw.Close()
}
}
func (b *Body) String() string {
b.readAll()
return string(b.full)
}
func (b *Body) Bytes() []byte {
b.readAll()
return b.full
}
func (b *Body) Unmarshal(u interface{}) error {
b.readAll()
return json.Unmarshal(b.full, u)
}
// Reader returns a reader for the body
// if this function is called, other functions like String, Bytes, Unmarshal may not work
func (b *Body) Reader() io.ReadCloser {
if b.isFull {
return io.NopCloser(bytes.NewReader(b.full))
}
b.isFull = true
return b.raw
}
func (b *Body) Close() error {
return b.Close()
}
func (r *Response) GetRequest() Request {
return r.req
}
func (r *Response) Body() *Body {
return r.data
}
func Curl(r *Request) (*Response, error) {
r.errInfo = nil
err := applyOptions(r)
if err != nil {
return nil, fmt.Errorf("apply options error: %s", err)
}
resp, err := r.rawClient.Do(r.rawRequest)
var res = Response{
Response: resp,
req: *r,
data: new(Body),
}
if err != nil {
res.Response = &http.Response{}
return &res, fmt.Errorf("do request error: %s", err)
}
res.data.raw = resp.Body
if r.autoFetchRespBody {
res.data.full, _ = io.ReadAll(resp.Body)
res.data.isFull = true
resp.Body.Close()
}
return &res, r.errInfo
}
func NewReq(uri string, opts ...RequestOpt) *Request {
return NewSimpleRequest(uri, "GET", opts...)
}
func NewReqWithContext(ctx context.Context, uri string, opts ...RequestOpt) *Request {
return NewSimpleRequestWithContext(ctx, uri, "GET", opts...)
}
func NewSimpleRequest(uri string, method string, opts ...RequestOpt) *Request {
r, _ := newRequest(context.Background(), uri, method, opts...)
return r
}
func NewRequest(uri string, method string, opts ...RequestOpt) (*Request, error) {
return newRequest(context.Background(), uri, method, opts...)
}
func NewSimpleRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) *Request {
r, _ := newRequest(ctx, uri, method, opts...)
return r
}
func NewRequestWithContext(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) {
return newRequest(ctx, uri, method, opts...)
}
func newRequest(ctx context.Context, uri string, method string, opts ...RequestOpt) (*Request, error) {
var req *http.Request
var err error
if method == "" {
method = "GET"
}
method = strings.ToUpper(method)
req, err = http.NewRequestWithContext(ctx, method, uri, nil)
if err != nil {
return nil, err
}
var r = &Request{
ctx: ctx,
uri: uri,
method: method,
RequestOpts: RequestOpts{
rawRequest: req,
rawClient: new(http.Client),
timeout: DefaultTimeout,
dialTimeout: DefaultDialTimeout,
autoFetchRespBody: DefaultFetchRespBody,
lookUpIPfn: net.DefaultResolver.LookupIPAddr,
bodyFormData: make(map[string][]string),
queries: make(map[string][]string),
},
}
r.headers = make(http.Header)
if strings.ToUpper(method) == "POST" {
r.headers.Set("Content-Type", HEADER_FORM_URLENCODE)
}
r.headers.Set("User-Agent", "B612 / 1.2.0")
for _, v := range opts {
if v != nil {
err = v(&r.RequestOpts)
if err != nil {
return nil, err
}
}
}
if r.transport == nil {
r.transport = &http.Transport{}
}
if r.doRawTransport {
if r.skipTLSVerify {
if r.transport.TLSClientConfig == nil {
r.transport.TLSClientConfig = &tls.Config{}
}
r.transport.TLSClientConfig.InsecureSkipVerify = true
}
if r.tlsConfig != nil {
r.transport.TLSClientConfig = r.tlsConfig
}
r.transport.DialContext = func(ctx context.Context, netType, addr string) (net.Conn, error) {
var lastErr error
var addrs []string
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
if len(r.customIP) > 0 {
for _, v := range r.customIP {
ipAddr := net.ParseIP(v)
if ipAddr == nil {
return nil, fmt.Errorf("invalid custom ip: %s", r.customIP)
}
tmpAddr := net.JoinHostPort(v, port)
addrs = append(addrs, tmpAddr)
}
} else {
ipLists, err := r.lookUpIPfn(ctx, host)
if err != nil {
return nil, err
}
for _, v := range ipLists {
tmpAddr := net.JoinHostPort(v.String(), port)
addrs = append(addrs, tmpAddr)
}
}
for _, addr := range addrs {
c, err := net.DialTimeout(netType, addr, r.dialTimeout)
if err != nil {
lastErr = err
continue
}
if r.timeout != 0 {
err = c.SetDeadline(time.Now().Add(r.timeout))
}
return c, nil
}
return nil, lastErr
}
}
return r, nil
}
func applyDataReader(r *Request) error {
// 优先度为bodyDataReader > bodyDataBytes > bodyFormData > bodyFileData
if r.bodyDataReader != nil {
r.rawRequest.Body = io.NopCloser(r.bodyDataReader)
return nil
}
if len(r.bodyDataBytes) != 0 {
r.rawRequest.Body = io.NopCloser(bytes.NewReader(r.bodyDataBytes))
return nil
}
if len(r.bodyFormData) != 0 && len(r.bodyFileData) == 0 {
var body = url.Values{}
for k, v := range r.bodyFormData {
for _, vv := range v {
body.Add(k, vv)
}
}
r.rawRequest.Body = io.NopCloser(strings.NewReader(body.Encode()))
return nil
}
if len(r.bodyFileData) != 0 {
var pr, pw = io.Pipe()
var w = multipart.NewWriter(pw)
r.rawRequest.Header.Set("Content-Type", w.FormDataContentType())
go func() {
defer pw.Close() // ensure pipe writer is closed
if len(r.bodyFormData) != 0 {
for k, v := range r.bodyFormData {
for _, vv := range v {
if err := w.WriteField(k, vv); err != nil {
r.errInfo = err
pw.CloseWithError(err) // close pipe with error
return
}
}
}
}
for _, v := range r.bodyFileData {
var fw, err = w.CreateFormFile(v.FormName, v.FileName)
if err != nil {
r.errInfo = err
pw.CloseWithError(err) // close pipe with error
return
}
if _, err := copyWithContext(r.ctx, r.FileUploadRecallFn, v.FileName, v.FileSize, fw, v.FileData); err != nil {
r.errInfo = err
pw.CloseWithError(err) // close pipe with error
return
}
}
if err := w.Close(); err != nil {
pw.CloseWithError(err) // close pipe with error if writer close fails
}
}()
r.rawRequest.Body = pr
return nil
}
return nil
}
func applyOptions(r *Request) error {
defer func() {
r.alreadyApply = true
}()
var req = r.rawRequest
if !r.doRawRequest {
if r.queries != nil {
sid := req.URL.Query()
for k, v := range r.queries {
for _, vv := range v {
sid.Add(k, vv)
}
}
req.URL.RawQuery = sid.Encode()
}
for k, v := range r.headers {
for _, vv := range v {
req.Header.Add(k, vv)
}
}
if len(r.cookies) != 0 {
for _, v := range r.cookies {
req.AddCookie(v)
}
}
if r.basicAuth[0] != "" || r.basicAuth[1] != "" {
req.SetBasicAuth(r.basicAuth[0], r.basicAuth[1])
}
err := applyDataReader(r)
if err != nil {
return fmt.Errorf("apply data reader error: %s", err)
}
if r.autoCalcContentLength {
if req.Body != nil {
data, err := io.ReadAll(req.Body)
if err != nil {
return fmt.Errorf("read data error: %s", err)
}
req.Header.Set("Content-Length", strconv.Itoa(len(data)))
req.Body = io.NopCloser(bytes.NewReader(data))
}
}
}
if r.proxy != "" {
purl, err := url.Parse(r.proxy)
if err != nil {
return fmt.Errorf("parse proxy url error: %s", err)
}
r.transport.Proxy = http.ProxyURL(purl)
}
if !r.doRawClient {
if !r.doRawTransport {
if !r.alreadySetLookUpIPfn && len(r.customIP) > 0 {
resolver := net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, address string) (conn net.Conn, err error) {
for _, addr := range r.customIP {
if conn, err = net.Dial("udp", addr+":53"); err != nil {
continue
} else {
return conn, nil
}
}
return
},
}
r.lookUpIPfn = resolver.LookupIPAddr
}
}
r.rawClient.Transport = r.transport
if r.disableRedirect {
r.rawClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
}
}
return nil
}
func copyWithContext(ctx context.Context, recall func(string, int64, int64), filename string, total int64, dst io.Writer, src io.Reader) (written int64, err error) {
pr, pw := io.Pipe()
defer pr.Close()
go func() {
defer pw.Close()
_, err := io.Copy(pw, src)
if err != nil {
pw.CloseWithError(err)
}
}()
var count int64
buf := make([]byte, 4096)
for {
select {
case <-ctx.Done():
return written, ctx.Err()
default:
nr, err := pr.Read(buf)
if err != nil {
if err == io.EOF {
go recall(filename, count, total)
return written, nil
}
return written, err
}
count += int64(nr)
if recall != nil {
go recall(filename, count, total)
}
nw, err := dst.Write(buf[:nr])
if err != nil {
return written, err
}
if nr != nw {
return written, io.ErrShortWrite
}
written += int64(nr)
}
}
}