You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
starnet/curl.go

463 lines
10 KiB
Go

package starnet
import (
"bytes"
"context"
"crypto/rand"
"crypto/tls"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
"b612.me/stario"
)
const (
HEADER_FORM_URLENCODE = `application/x-www-form-urlencoded`
HEADER_FORM_DATA = `multipart/form-data`
HEADER_JSON = `application/json`
)
type RequestFile struct {
UploadFile string
UploadForm map[string]string
UploadName string
}
type Request struct {
Url string
RespURL string
Method string
RecvData []byte
RecvContentLength int64
RecvIo io.Writer
RespHeader http.Header
RespCookies []*http.Cookie
RespHttpCode int
Location *url.URL
CircleBuffer *stario.StarBuffer
respReader io.ReadCloser
respOrigin *http.Response
reqOrigin *http.Request
RequestOpts
}
type RequestOpts struct {
RequestFile
PostBuffer io.Reader
Process func(float64)
Proxy string
Timeout time.Duration
DialTimeout time.Duration
ReqHeader http.Header
ReqCookies []*http.Cookie
WriteRecvData bool
SkipTLSVerify bool
CustomTransport *http.Transport
Queries map[string]string
DisableRedirect bool
TlsConfig *tls.Config
}
type RequestOpt func(opt *RequestOpts)
func WithDialTimeout(timeout time.Duration) RequestOpt {
return func(opt *RequestOpts) {
opt.DialTimeout = timeout
}
}
func WithTimeout(timeout time.Duration) RequestOpt {
return func(opt *RequestOpts) {
opt.Timeout = timeout
}
}
func WithHeader(key, val string) RequestOpt {
return func(opt *RequestOpts) {
opt.ReqHeader.Set(key, val)
}
}
func WithTlsConfig(tlscfg *tls.Config) RequestOpt {
return func(opt *RequestOpts) {
opt.TlsConfig = tlscfg
}
}
func WithHeaderMap(header map[string]string) RequestOpt {
return func(opt *RequestOpts) {
for key, val := range header {
opt.ReqHeader.Set(key, val)
}
}
}
func WithHeaderAdd(key, val string) RequestOpt {
return func(opt *RequestOpts) {
opt.ReqHeader.Add(key, val)
}
}
func WithReader(r io.Reader) RequestOpt {
return func(opt *RequestOpts) {
opt.PostBuffer = r
}
}
func WithFetchRespBody(fetch bool) RequestOpt {
return func(opt *RequestOpts) {
opt.WriteRecvData = fetch
}
}
func WithCookies(ck []*http.Cookie) RequestOpt {
return func(opt *RequestOpts) {
opt.ReqCookies = ck
}
}
func WithCookie(key, val, path string) RequestOpt {
return func(opt *RequestOpts) {
opt.ReqCookies = append(opt.ReqCookies, &http.Cookie{Name: key, Value: val, Path: path})
}
}
func WithCookieMap(header map[string]string, path string) RequestOpt {
return func(opt *RequestOpts) {
for key, val := range header {
opt.ReqCookies = append(opt.ReqCookies, &http.Cookie{Name: key, Value: val, Path: path})
}
}
}
func WithQueries(queries map[string]string) RequestOpt {
return func(opt *RequestOpts) {
opt.Queries = queries
}
}
func WithProxy(proxy string) RequestOpt {
return func(opt *RequestOpts) {
opt.Proxy = proxy
}
}
func WithProcess(fn func(float64)) RequestOpt {
return func(opt *RequestOpts) {
opt.Process = fn
}
}
func WithContentType(ct string) RequestOpt {
return func(opt *RequestOpts) {
opt.ReqHeader.Set("Content-Type", ct)
}
}
func WithUserAgent(ua string) RequestOpt {
return func(opt *RequestOpts) {
opt.ReqHeader.Set("User-Agent", ua)
}
}
func WithCustomTransport(hs *http.Transport) RequestOpt {
return func(opt *RequestOpts) {
opt.CustomTransport = hs
}
}
func WithSkipTLSVerify(skip bool) RequestOpt {
return func(opt *RequestOpts) {
opt.SkipTLSVerify = skip
}
}
func WithDisableRedirect(disable bool) RequestOpt {
return func(opt *RequestOpts) {
opt.DisableRedirect = disable
}
}
func NewRequests(url string, rawdata []byte, method string, opts ...RequestOpt) Request {
req := Request{
RequestOpts: RequestOpts{
Timeout: 30 * time.Second,
DialTimeout: 15 * time.Second,
WriteRecvData: true,
},
Url: url,
Method: method,
}
if rawdata != nil {
req.PostBuffer = bytes.NewBuffer(rawdata)
}
req.ReqHeader = make(http.Header)
if strings.ToUpper(method) == "POST" {
req.ReqHeader.Set("Content-Type", HEADER_FORM_URLENCODE)
}
req.ReqHeader.Set("User-Agent", "B612 / 1.1.0")
for _, v := range opts {
v(&req.RequestOpts)
}
if req.CustomTransport == nil {
req.CustomTransport = &http.Transport{}
}
if req.SkipTLSVerify {
if req.CustomTransport.TLSClientConfig == nil {
req.CustomTransport.TLSClientConfig = &tls.Config{}
}
req.CustomTransport.TLSClientConfig.InsecureSkipVerify = true
}
if req.TlsConfig != nil {
req.CustomTransport.TLSClientConfig = req.TlsConfig
}
req.CustomTransport.DialContext = func(ctx context.Context, netw, addr string) (net.Conn, error) {
c, err := net.DialTimeout(netw, addr, req.DialTimeout)
if err != nil {
return nil, err
}
if req.Timeout != 0 {
c.SetDeadline(time.Now().Add(req.Timeout))
}
return c, nil
}
return req
}
func (curl *Request) ResetReqHeader() {
curl.ReqHeader = make(http.Header)
}
func (curl *Request) ResetReqCookies() {
curl.ReqCookies = []*http.Cookie{}
}
func (curl *Request) AddSimpleCookie(key, value string) {
curl.ReqCookies = append(curl.ReqCookies, &http.Cookie{Name: key, Value: value, Path: "/"})
}
func (curl *Request) AddCookie(key, value, path string) {
curl.ReqCookies = append(curl.ReqCookies, &http.Cookie{Name: key, Value: value, Path: path})
}
func randomBoundary() string {
var buf [30]byte
_, err := io.ReadFull(rand.Reader, buf[:])
if err != nil {
panic(err)
}
return fmt.Sprintf("%x", buf[:])
}
func Curl(curl Request) (resps Request, err error) {
var fpsrc *os.File
if curl.RequestFile.UploadFile != "" {
fpsrc, err = os.Open(curl.UploadFile)
if err != nil {
return
}
defer fpsrc.Close()
boundary := randomBoundary()
boundarybytes := []byte("\r\n--" + boundary + "\r\n")
endbytes := []byte("\r\n--" + boundary + "--\r\n")
fpstat, _ := fpsrc.Stat()
filebig := float64(fpstat.Size())
sum, n := 0, 0
fpdst := stario.NewStarBuffer(1048576)
if curl.UploadForm != nil {
for k, v := range curl.UploadForm {
header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\";\r\nContent-Type: x-www-form-urlencoded \r\n\r\n", k)
fpdst.Write(boundarybytes)
fpdst.Write([]byte(header))
fpdst.Write([]byte(v))
}
}
header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\nContent-Type: application/octet-stream\r\n\r\n", curl.UploadName, fpstat.Name())
fpdst.Write(boundarybytes)
fpdst.Write([]byte(header))
go func() {
for {
bufs := make([]byte, 393213)
n, err = fpsrc.Read(bufs)
if err != nil {
if err == io.EOF {
if n != 0 {
fpdst.Write(bufs[0:n])
if curl.Process != nil {
go curl.Process(float64(sum+n) / filebig * 100)
}
}
break
}
return
}
sum += n
if curl.Process != nil {
go curl.Process(float64(sum+n) / filebig * 100)
}
fpdst.Write(bufs[0:n])
}
fpdst.Write(endbytes)
fpdst.Write(nil)
}()
curl.CircleBuffer = fpdst
curl.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+boundary)
}
req, resp, err := netcurl(curl)
if err != nil {
return Request{}, err
}
if resp.Request != nil && resp.Request.URL != nil {
curl.RespURL = resp.Request.URL.String()
}
curl.reqOrigin = req
curl.respOrigin = resp
curl.Location, _ = resp.Location()
curl.RespHttpCode = resp.StatusCode
curl.RespHeader = resp.Header
curl.RespCookies = resp.Cookies()
curl.RecvContentLength = resp.ContentLength
readFunc := func(reader io.ReadCloser, writer io.Writer) error {
lengthall := resp.ContentLength
defer reader.Close()
var lengthsum int
buf := make([]byte, 65535)
for {
n, err := reader.Read(buf)
if n != 0 {
_, err := writer.Write(buf[:n])
lengthsum += n
if curl.Process != nil {
go curl.Process(float64(lengthsum) / float64(lengthall) * 100.00)
}
if err != nil {
return err
}
}
if err != nil && err != io.EOF {
return err
} else if err == io.EOF {
return nil
}
}
}
if curl.WriteRecvData {
buf := bytes.NewBuffer([]byte{})
err = readFunc(resp.Body, buf)
if err != nil {
return
}
curl.RecvData = buf.Bytes()
} else {
curl.respReader = resp.Body
}
if curl.RecvIo != nil {
if curl.WriteRecvData {
_, err = curl.RecvIo.Write(curl.RecvData)
} else {
err = readFunc(resp.Body, curl.RecvIo)
if err != nil {
return
}
}
}
return curl, err
}
// RespBodyReader Only works when WriteRecvData set to false
func (curl *Request) RespBodyReader() io.ReadCloser {
return curl.respReader
}
func netcurl(curl Request) (*http.Request, *http.Response, error) {
var req *http.Request
var err error
if curl.Method == "" {
return nil, nil, errors.New("Error Method Not Entered")
}
if curl.PostBuffer != nil {
req, err = http.NewRequest(curl.Method, curl.Url, curl.PostBuffer)
} else if curl.CircleBuffer != nil && curl.CircleBuffer.Len() > 0 {
req, err = http.NewRequest(curl.Method, curl.Url, curl.CircleBuffer)
} else {
req, err = http.NewRequest(curl.Method, curl.Url, nil)
}
if curl.Queries != nil {
sid := req.URL.Query()
for k, v := range curl.Queries {
sid.Add(k, v)
}
req.URL.RawQuery = sid.Encode()
}
if err != nil {
return nil, nil, err
}
req.Header = curl.ReqHeader
if len(curl.ReqCookies) != 0 {
for _, v := range curl.ReqCookies {
req.AddCookie(v)
}
}
if curl.Proxy != "" {
purl, err := url.Parse(curl.Proxy)
if err != nil {
return nil, nil, err
}
curl.CustomTransport.Proxy = http.ProxyURL(purl)
}
client := &http.Client{
Transport: curl.CustomTransport,
}
if curl.DisableRedirect {
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
}
resp, err := client.Do(req)
return req, resp, err
}
func UrlEncodeRaw(str string) string {
strs := strings.Replace(url.QueryEscape(str), "+", "%20", -1)
return strs
}
func UrlEncode(str string) string {
return url.QueryEscape(str)
}
func UrlDecode(str string) (string, error) {
return url.QueryUnescape(str)
}
func BuildQuery(queryData map[string]string) string {
query := url.Values{}
for k, v := range queryData {
query.Add(k, v)
}
return query.Encode()
}
func BuildPostForm(queryMap map[string]string) []byte {
query := url.Values{}
for k, v := range queryMap {
query.Add(k, v)
}
return []byte(query.Encode())
}
func (r Request) Resopnse() *http.Response {
return r.respOrigin
}
func (r Request) Request() *http.Request {
return r.reqOrigin
}