package fetch import ( "bytes" "context" "crypto/tls" "encoding/json" "encoding/xml" "fmt" "git.nspix.com/golang/kos/util/env" "io" "net" "net/http" "net/url" "path" "strings" "time" ) var ( httpClient = http.Client{ Timeout: time.Second * 30, Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, ForceAttemptHTTP2: false, MaxIdleConns: 48, IdleConnTimeout: 30 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, } ) func init() { httpDefaultTimeout := env.Get("HTTP_CLIENT_TIMEOUT", "30s") if httpDefaultTimeout != "" { if duration, err := time.ParseDuration(httpDefaultTimeout); err == nil { httpClient.Timeout = duration } } } func encode(data any) (r io.Reader, contentType string, err error) { var ( buf []byte ) switch v := data.(type) { case string: r = strings.NewReader(v) contentType = "x-www-form-urlencoded" case []byte: r = bytes.NewReader(v) contentType = "x-www-form-urlencoded" default: if buf, err = json.Marshal(v); err == nil { r = bytes.NewReader(buf) contentType = "application/json" } } return } func Get(ctx context.Context, urlString string, cbs ...Option) (res *http.Response, err error) { var ( uri *url.URL req *http.Request ) opts := newOptions() for _, cb := range cbs { cb(opts) } if uri, err = url.Parse(urlString); err != nil { return } if opts.Params != nil { qs := uri.Query() for k, v := range opts.Params { qs.Set(k, v) } uri.RawQuery = qs.Encode() } if req, err = http.NewRequest(http.MethodGet, uri.String(), nil); err != nil { return } if opts.Header != nil { for k, v := range opts.Header { req.Header.Set(k, v) } } return do(ctx, req, opts) } func Post(ctx context.Context, urlString string, cbs ...Option) (res *http.Response, err error) { var ( uri *url.URL req *http.Request contentType string reader io.Reader ) opts := newOptions() for _, cb := range cbs { cb(opts) } if uri, err = url.Parse(urlString); err != nil { return } if opts.Params != nil { qs := uri.Query() for k, v := range opts.Params { qs.Set(k, v) } uri.RawQuery = qs.Encode() } if opts.Data != nil { if reader, contentType, err = encode(opts.Data); err != nil { return } } if req, err = http.NewRequest(http.MethodPost, uri.String(), reader); err != nil { return } if opts.Header != nil { for k, v := range opts.Header { req.Header.Set(k, v) } } if contentType != "" { req.Header.Set("Content-Type", contentType) } return do(ctx, req, opts) } func Echo(ctx context.Context, method, uri string, response any, cbs ...Option) (err error) { cbs = append(cbs, WithMethod(method)) return Request(ctx, uri, response, cbs...) } func Request(ctx context.Context, u string, response any, cbs ...Option) (err error) { var ( buf []byte uri *url.URL res *http.Response req *http.Request contentType string reader io.Reader ) opts := newOptions() for _, cb := range cbs { cb(opts) } if uri, err = url.Parse(u); err != nil { return } if opts.Params != nil { qs := uri.Query() for k, v := range opts.Params { qs.Set(k, v) } uri.RawQuery = qs.Encode() } if opts.Data != nil { if reader, contentType, err = encode(opts.Data); err != nil { return } } if req, err = http.NewRequest(opts.Method, uri.String(), reader); err != nil { return } if opts.Header != nil { for k, v := range opts.Header { req.Header.Set(k, v) } } if contentType != "" { req.Header.Set("Content-Type", contentType) } if res, err = do(ctx, req, opts); err != nil { return } defer func() { _ = res.Body.Close() }() if res.StatusCode != http.StatusOK { if buf, err = io.ReadAll(res.Body); err == nil && len(buf) > 0 { err = fmt.Errorf("remote server response %s(%d): %s", res.Status, res.StatusCode, string(buf)) } else { err = fmt.Errorf("remote server response %d: %s", res.StatusCode, res.Status) } return } //don't care response if response == nil { return } contentType = strings.ToLower(res.Header.Get("Content-Type")) extName := path.Ext(req.URL.String()) if strings.Contains(contentType, JSON) || extName == ".json" { err = json.NewDecoder(res.Body).Decode(response) } else if strings.Contains(contentType, XML) || extName == ".xml" { err = xml.NewDecoder(res.Body).Decode(response) } else { err = fmt.Errorf("unsupported content type: %s", contentType) } return } func Do(ctx context.Context, req *http.Request, cbs ...Option) (res *http.Response, err error) { opts := newOptions() for _, cb := range cbs { cb(opts) } return do(ctx, req, opts) } func do(ctx context.Context, req *http.Request, opts *Options) (res *http.Response, err error) { if opts.Human { if req.Header.Get("User-Agent") == "" { req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54") } if req.Header.Get("Referer") == "" { req.Header.Set("Referer", req.URL.String()) } if req.Header.Get("Accept") == "" { req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") } } return httpClient.Do(req.WithContext(ctx)) }