add httpclient fetch

helper/httpclient/client.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"encoding/xml"
+	"fmt"
 	"git.nspix.com/golang/micro/helper/random"
 	"io"
 	"net"
@@ -92,6 +94,40 @@ func MockDo(ctx context.Context, req *http.Request) (res *http.Response, err err
 	return Do(ctx, req)
 }
 
+func Fetch(ctx context.Context, req *http.Request, outer interface{}) (err error) {
+	var (
+		res         *http.Response
+		contentType string
+	)
+	randomIp := random.IP()
+	req.Header.Set("X-Forwarded-For", randomIp)
+	req.Header.Set("X-Real-IP", randomIp)
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.81 Safari/537.36 Edg/104.0.1293.54")
+	if req.Header.Get("Referer") == "" {
+		req.Header.Set("Referer", req.URL.String())
+	}
+	if ctx != nil {
+		req = req.WithContext(ctx)
+	}
+	if res, err = Do(ctx, req); err != nil {
+		return
+	}
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	if res.StatusCode != http.StatusOK {
+		err = fmt.Errorf("got http response (%d)%s", res.StatusCode, res.Status)
+		return
+	}
+	contentType = res.Header.Get("Content-Type")
+	if strings.HasPrefix(contentType, "application/json") {
+		err = json.NewDecoder(res.Body).Decode(outer)
+	} else if strings.HasPrefix(contentType, "application/xml") {
+		err = xml.NewDecoder(res.Body).Decode(outer)
+	}
+	return
+}
+
 func Do(ctx context.Context, req *http.Request) (res *http.Response, err error) {
 	if ctx != nil {
 		req = req.WithContext(ctx)