starnet/http.go

package starnet

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net"
	"net/http"
	urls "net/url"
	"os"
	"path/filepath"
	"time"
)

type StarCurl struct {
	TimeOut      int
	DialTimeOut  int
	ReqHeader    http.Header
	ReqCookies   []*http.Cookie
	RespHeader   http.Header
	RespCookies  []*http.Cookie
	RespHttpCode int
	PostBuffer   *bytes.Buffer
	CircleBuffer *CircleByteBuffer
	Proxy        string
}

func NewStarCurl() *StarCurl {
	star := new(StarCurl)
	star.ReqHeader = make(http.Header)
	star.TimeOut = 60
	star.DialTimeOut = 15
	star.PostBuffer = nil
	return star
}

func (this *StarCurl) ResetReqHeader() {
	this.ReqHeader = make(http.Header)
}

func (this *StarCurl) ResetReqCookies() {
	this.ReqCookies = []*http.Cookie{}
}

func (this *StarCurl) AddSimpleCookie(key, value string) {
	this.ReqCookies = append(this.ReqCookies, &http.Cookie{Name: key, Value: value, Path: "/"})
}

func (this *StarCurl) CurlWithFile(url string, postdata map[string]string, formname, fpath, savepath string, tofile bool, shell func(float64)) (result []byte, err error) {
	fpsrc, err := os.Open(fpath)
	if err != nil {
		return
	}
	defer fpsrc.Close()
	boundary := randomBoundary()
	boundarybytes := []byte("\r\n--" + boundary + "\r\n")
	endbytes := []byte("\r\n--" + boundary + "--\r\n")
	fpstat, _ := os.Stat(fpath)
	filebig := float64(fpstat.Size())
	sum, n := 0, 0
	fpdst := NewCircleByteBuffer(1048576)
	if postdata != nil {
		for k, v := range postdata {
			header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\";\r\nContent-Type: x-www-form-urlencoded \r\n\r\n", k)
			fpdst.Write(boundarybytes)
			fpdst.Write([]byte(header))
			fpdst.Write([]byte(v))
		}
	}
	header := fmt.Sprintf("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\nContent-Type: application/octet-stream\r\n\r\n", formname, fpstat.Name())
	fpdst.Write(boundarybytes)
	fpdst.Write([]byte(header))
	go func() {
		for {
			bufs := make([]byte, 393213)
			n, err = fpsrc.Read(bufs)
			if err != nil {
				if err == io.EOF {
					if n != 0 {
						fpdst.Write(bufs[0:n])
						go shell(float64(sum+n) / filebig * 100)
					}
					break
				}
				return
			}
			sum += n
			go shell(float64(sum) / filebig * 100)
			fpdst.Write(bufs[0:n])
		}
		fpdst.Write(endbytes)
		fpdst.Write(nil)
	}()
	this.CircleBuffer = fpdst
	this.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+boundary)
	if tofile {
		err = this.CurlDataToFile(url, []byte{}, "POST", savepath, shell)
		this.ResetReqHeader()
	} else {
		result, err = this.Curl(url, []byte{}, "POST")
	}
	this.ResetReqHeader()
	return
}

func (this *StarCurl) CurlWithFileByBytes(url string, postdata map[string]string, formname, fname string, data []byte, savepath string, tofile bool) (result []byte, err error) {
	buf := &bytes.Buffer{}
	bufwriter := multipart.NewWriter(buf)
	if postdata != nil {
		for k, v := range postdata {
			bufwriter.WriteField(k, v)
		}
	}
	fpdst, err := bufwriter.CreateFormFile(formname, fname)
	if err != nil {
		return
	}
	fpdst.Write(data)
	this.PostBuffer = buf
	this.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+bufwriter.Boundary())
	bufwriter.Close()
	if tofile {
		err = this.CurlDataToFile(url, []byte{}, "POST", savepath, func(float64) {})
		this.ResetReqHeader()
	} else {
		result, err = this.Curl(url, []byte{}, "POST")
	}
	this.ResetReqHeader()
	return
}

func (this *StarCurl) CurlWithFileByMemory(url string, postdata map[string]string, formname, fpath, savepath string, tofile bool, shell func(float64)) (result []byte, err error) {
	buf := &bytes.Buffer{}
	bufwriter := multipart.NewWriter(buf)
	if postdata != nil {
		for k, v := range postdata {
			bufwriter.WriteField(k, v)
		}
	}
	fpdst, err := bufwriter.CreateFormFile(formname, filepath.Base(fpath))
	if err != nil {
		return
	}
	fpsrc, err := os.Open(fpath)
	if err != nil {
		return
	}
	defer fpsrc.Close()
	fpstat, _ := os.Stat(fpath)
	filebig := float64(fpstat.Size())
	sum, n := 0, 0
	for {
		bufs := make([]byte, 393213)
		n, err = fpsrc.Read(bufs)
		if err != nil {
			if err == io.EOF {
				if n != 0 {
					fpdst.Write(bufs[0:n])
					go shell(float64(sum+n) / filebig * 100)
				}
				break
			}
			return
		}
		sum += n
		go shell(float64(sum) / filebig * 100)
		fpdst.Write(bufs[0:n])
	}

	this.PostBuffer = buf
	this.ReqHeader.Set("Content-Type", "multipart/form-data;boundary="+bufwriter.Boundary())
	bufwriter.Close()
	if tofile {
		err = this.CurlDataToFile(url, []byte{}, "POST", savepath, shell)
		this.ResetReqHeader()
	} else {
		result, err = this.Curl(url, []byte{}, "POST")
	}
	this.ResetReqHeader()
	return
}

func (this *StarCurl) CurlDataToFile(url string, postdata []byte, method, fpath string, shell func(float64)) (err error) {
	var req *http.Request
	if method == "" {
		if len(postdata) != 0 {
			method = "POST"
		} else {
			method = "GET"
		}
	}
	if len(postdata) == 0 && this.PostBuffer == nil && this.CircleBuffer == nil {
		req, err = http.NewRequest(method, url, nil)
	} else if len(postdata) != 0 {
		req, err = http.NewRequest(method, url, bytes.NewBuffer(postdata))
	} else if this.PostBuffer != nil {
		req, err = http.NewRequest(method, url, this.PostBuffer)
	} else {
		req, err = http.NewRequest(method, url, this.CircleBuffer)
	}
	if err != nil {
		return
	}
	if (this.ReqHeader == nil) && method == "POST" {
		this.ReqHeader.Set("Content-Type", "application/x-www-form-urlencoded")
	}
	req.Header = this.ReqHeader
	if len(this.ReqCookies) != 0 {
		for _, v := range this.ReqCookies {
			req.AddCookie(v)
		}
	}
	transport := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			deadline := time.Now().Add(time.Duration(this.TimeOut) * time.Second)
			c, err := net.DialTimeout(netw, addr, time.Second*time.Duration(this.DialTimeOut))
			if err != nil {
				return nil, err
			}
			if this.TimeOut != 0 {
				c.SetDeadline(deadline)
			}
			return c, nil
		},
	}
	if this.Proxy != "" {
		purl, _ := urls.Parse(this.Proxy)
		transport.Proxy = http.ProxyURL(purl)
	}
	client := &http.Client{
		Transport: transport,
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	this.PostBuffer = nil
	this.CircleBuffer = nil
	this.RespHttpCode = resp.StatusCode
	this.RespHeader = resp.Header
	this.RespCookies = resp.Cookies()
	fpsrc, err := os.Create(fpath)
	if err != nil {
		return
	}
	defer fpsrc.Close()
	filebig := float64(resp.ContentLength)
	if filebig <= 0 {
		filebig = 100
	}
	var n, sum int = 0, 0
	for {
		buf := make([]byte, 393213)
		n, err = resp.Body.Read(buf)
		if err != nil {
			if err == io.EOF {
				err = nil
				if n != 0 {
					fpsrc.Write(buf[0:n])
				}
				go shell(100.00)
				break
			}
			return
		}
		sum += n
		go shell(float64(sum) / filebig * 100.00)
		fpsrc.Write(buf[0:n])
	}
	return
}

func (this *StarCurl) Curl(url string, postdata []byte, method string) (body []byte, err error) {
	var req *http.Request
	if method == "" {
		if len(postdata) != 0 {
			method = "POST"
		} else {
			method = "GET"
		}
	}
	if len(postdata) == 0 && this.PostBuffer == nil && this.CircleBuffer == nil {
		req, err = http.NewRequest(method, url, nil)
	} else if len(postdata) != 0 {
		req, err = http.NewRequest(method, url, bytes.NewBuffer(postdata))
	} else if this.PostBuffer != nil {
		req, err = http.NewRequest(method, url, this.PostBuffer)
	} else {
		req, err = http.NewRequest(method, url, this.CircleBuffer)
	}
	if err != nil {
		return
	}
	if (this.ReqHeader == nil) && method == "POST" {
		this.ReqHeader.Set("Content-Type", "application/x-www-form-urlencoded")
	}
	req.Header = this.ReqHeader
	if len(this.ReqCookies) != 0 {
		for _, v := range this.ReqCookies {
			req.AddCookie(v)
		}
	}
	transport := &http.Transport{
		Dial: func(netw, addr string) (net.Conn, error) {
			deadline := time.Now().Add(time.Duration(this.TimeOut) * time.Second)
			c, err := net.DialTimeout(netw, addr, time.Second*time.Duration(this.DialTimeOut))
			if err != nil {
				return nil, err
			}
			if this.TimeOut != 0 {
				c.SetDeadline(deadline)
			}
			return c, nil
		},
	}
	if this.Proxy != "" {
		purl, _ := urls.Parse(this.Proxy)
		transport.Proxy = http.ProxyURL(purl)
	}
	client := &http.Client{
		Transport: transport,
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	this.PostBuffer = nil
	this.CircleBuffer = nil
	this.RespHttpCode = resp.StatusCode
	this.RespHeader = resp.Header
	this.RespCookies = resp.Cookies()
	body, err = ioutil.ReadAll(resp.Body)
	return
}