kos/util/fetch/fetch.go

package fetch

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"git.nspix.com/golang/kos/util/env"
	"io"
	"net"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

var (
	httpClient = http.Client{
		Timeout: time.Second * 30,
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).DialContext,
			ForceAttemptHTTP2:     false,
			MaxIdleConns:          48,
			IdleConnTimeout:       30 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}
)

func init() {
	httpDefaultTimeout := env.Get("HTTP_CLIENT_TIMEOUT", "30s")
	if httpDefaultTimeout != "" {
		if duration, err := time.ParseDuration(httpDefaultTimeout); err == nil {
			httpClient.Timeout = duration
		}
	}
}

func encode(data any) (r io.Reader, contentType string, err error) {
	var (
		buf []byte
	)
	switch v := data.(type) {
	case string:
		r = strings.NewReader(v)
		contentType = "x-www-form-urlencoded"
	case []byte:
		r = bytes.NewReader(v)
		contentType = "x-www-form-urlencoded"
	default:
		if buf, err = json.Marshal(v); err == nil {
			r = bytes.NewReader(buf)
			contentType = "application/json"
		}
	}
	return
}

func Get(ctx context.Context, urlString string, cbs ...Option) (res *http.Response, err error) {
	var (
		uri *url.URL
		req *http.Request
	)
	opts := newOptions()
	for _, cb := range cbs {
		cb(opts)
	}
	if uri, err = url.Parse(urlString); err != nil {
		return
	}
	if opts.Params != nil {
		qs := uri.Query()
		for k, v := range opts.Params {
			qs.Set(k, v)
		}
		uri.RawQuery = qs.Encode()
	}
	if req, err = http.NewRequest(http.MethodGet, uri.String(), nil); err != nil {
		return
	}
	if opts.Header != nil {
		for k, v := range opts.Header {
			req.Header.Set(k, v)
		}
	}
	return do(ctx, req, opts)
}

func Post(ctx context.Context, urlString string, cbs ...Option) (res *http.Response, err error) {
	var (
		uri         *url.URL
		req         *http.Request
		contentType string
		reader      io.Reader
	)
	opts := newOptions()
	for _, cb := range cbs {
		cb(opts)
	}
	if uri, err = url.Parse(urlString); err != nil {
		return
	}
	if opts.Params != nil {
		qs := uri.Query()
		for k, v := range opts.Params {
			qs.Set(k, v)
		}
		uri.RawQuery = qs.Encode()
	}
	if opts.Data != nil {
		if reader, contentType, err = encode(opts.Data); err != nil {
			return
		}
	}
	if req, err = http.NewRequest(http.MethodPost, uri.String(), reader); err != nil {
		return
	}
	if opts.Header != nil {
		for k, v := range opts.Header {
			req.Header.Set(k, v)
		}
	}
	if contentType != "" {
		req.Header.Set("Content-Type", contentType)
	}
	return do(ctx, req, opts)
}

func Echo(ctx context.Context, method, uri string, response any, cbs ...Option) (err error) {
	cbs = append(cbs, WithMethod(method))
	return Request(ctx, uri, response, cbs...)
}

func Request(ctx context.Context, u string, response any, cbs ...Option) (err error) {
	var (
		buf         []byte
		uri         *url.URL
		res         *http.Response
		req         *http.Request
		contentType string
		reader      io.Reader
	)
	opts := newOptions()
	for _, cb := range cbs {
		cb(opts)
	}
	if uri, err = url.Parse(u); err != nil {
		return
	}
	if opts.Params != nil {
		qs := uri.Query()
		for k, v := range opts.Params {
			qs.Set(k, v)
		}
		uri.RawQuery = qs.Encode()
	}
	if opts.Data != nil {
		if reader, contentType, err = encode(opts.Data); err != nil {
			return
		}
	}
	if req, err = http.NewRequest(opts.Method, uri.String(), reader); err != nil {
		return
	}
	if opts.Header != nil {
		for k, v := range opts.Header {
			req.Header.Set(k, v)
		}
	}
	if contentType != "" {
		req.Header.Set("Content-Type", contentType)
	}
	if res, err = do(ctx, req, opts); err != nil {
		return
	}
	defer func() {
		_ = res.Body.Close()
	}()
	if res.StatusCode != http.StatusOK {
		if buf, err = io.ReadAll(res.Body); err == nil && len(buf) > 0 {
			err = fmt.Errorf("remote server response %s(%d): %s", res.Status, res.StatusCode, string(buf))
		} else {
			err = fmt.Errorf("remote server response %d: %s", res.StatusCode, res.Status)
		}
		return
	}
	//don't care response
	if response == nil {
		return
	}
	contentType = strings.ToLower(res.Header.Get("Content-Type"))
	extName := path.Ext(req.URL.String())
	if strings.Contains(contentType, JSON) || extName == ".json" {
		err = json.NewDecoder(res.Body).Decode(response)
	} else if strings.Contains(contentType, XML) || extName == ".xml" {
		err = xml.NewDecoder(res.Body).Decode(response)
	} else {
		err = fmt.Errorf("unsupported content type: %s", contentType)
	}
	return
}

func Do(ctx context.Context, req *http.Request, cbs ...Option) (res *http.Response, err error) {
	opts := newOptions()
	for _, cb := range cbs {
		cb(opts)
	}
	return do(ctx, req, opts)
}

func do(ctx context.Context, req *http.Request, opts *Options) (res *http.Response, err error) {
	if opts.Human {
		if req.Header.Get("User-Agent") == "" {
			req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54")
		}
		if req.Header.Get("Referer") == "" {
			req.Header.Set("Referer", req.URL.String())
		}
		if req.Header.Get("Accept") == "" {
			req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
		}
	}
	return httpClient.Do(req.WithContext(ctx))
}