java-classic-code/httpclient/RestApiUtils.java

package com.yiidata.intergration.api.utils;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.datasophon.api.utils.HttpUtils;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.FileEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;


/**
 *
 * 封装 RestApi 调用
 *
 * <pre>
 *
 * Created by zhaopx.
 * User: zhaopx
 * Date: 2020/4/3
 * Time: 17:38
 *
 * </pre>
 *
 * @author zhaopx
 */
public class RestApiUtils {


    static Logger log = LoggerFactory.getLogger(RestApiUtils.class);

    /**
     * http client
     */
    static final CloseableHttpClient httpClient;


    /**
     * Cookie store
     */
    private final static BasicCookieStore cookieStore = new BasicCookieStore();

    static {
        try {
            //1. 设置协议http和https对应的处理socket链接工厂的对象
            Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                    .register("http", PlainConnectionSocketFactory.INSTANCE)
                    .register("https", new SSLConnectionSocketFactory(createIgnoreVerifySSL(), null, null, new DefaultHostnameVerifier()))
                    .build();

            //2. Socket层优化
            SocketConfig socketConfig = SocketConfig.custom()
                    .setTcpNoDelay(true)    // 禁用Nagle算法
                    .setSoKeepAlive(true)   // 启用TCP KeepAlive
                    .setSoReuseAddress(true)
                    .build();

            PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
            // 连接池大小
            cm.setMaxTotal(220);
            cm.setDefaultMaxPerRoute((int)(cm.getMaxTotal() * 0.8));
            cm.setDefaultSocketConfig(socketConfig);
            httpClient = HttpClients.custom()
                    .setConnectionManager(cm)
                    .setDefaultCookieStore(cookieStore)
                    .setDefaultRequestConfig(RequestConfig.custom()
                            .setConnectTimeout(65000)
                            .setSocketTimeout(30000)
                            .setConnectionRequestTimeout(30000).build())
                    .setRedirectStrategy(new DefaultRedirectStrategy() {
                        @Override
                        protected boolean isRedirectable(String method) {
                            // If the connection target is FE, you need to handle 307 redirect.
                            return true;
                        }
                    })
                    .setRetryHandler(new DefaultHttpRequestRetryHandler(3, false))
                    .setConnectionManagerShared(false) // 共享连接池
                    .evictIdleConnections(15, TimeUnit.SECONDS) // 定期清理空闲连接
                    .build();
        } catch (Exception e) {
            throw new IllegalStateException(e);
        }
    }


    /**
     * 绕过验证
     *
     * @return
     * @throws NoSuchAlgorithmException
     * @throws KeyManagementException
     */
    public static SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
        SSLContext sc = SSLContext.getInstance("TLSv1.2");

        // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
        X509TrustManager trustManager = new X509TrustManager() {
            @Override
            public void checkClientTrusted(
                    java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
                    String paramString) throws CertificateException {
            }

            @Override
            public void checkServerTrusted(
                    java.security.cert.X509Certificate[] paramArrayOfX509Certificate,
                    String paramString) throws CertificateException {
            }

            @Override
            public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                return null;
            }
        };

        sc.init(null, new TrustManager[] { trustManager }, null);
        return sc;
    }

    /**
     * 调用一次远程 api
     *
     * @param url
     * @return
     * @throws IOException
     */
    public static Map<String, Object> post(String url) throws IOException {
        return post(url, new HashMap<>(), false);
    }


    /**
     * 调用一次远程 api
     *
     * @param url
     * @return
     * @throws IOException
     */
    public static Map<String, Object> post(String url, boolean not200ThrowError) throws IOException {
        return post(url, new HashMap<>(), not200ThrowError);
    }


    /**
     * 通过 Post 请求调用Rest 接口
     *
     * @param url
     * @param json
     * @return
     * @throws IOException
     */
    public static Map<String, Object> post(String url, Map<String, Object> json) throws IOException {
        return post(url, json, false);
    }

    /**
     * 通过 Post 请求调用Rest 接口
     *
     * @param url
     * @param json
     * @param not200ThrowError 为 true 时，当返回不是 200，则抛出异常
     * @return
     * @throws IOException
     */
    public static Map<String, Object> post(String url, Map<String, Object> json, boolean not200ThrowError) throws IOException {
        return post(url, JSON.toJSONString(json), not200ThrowError);
    }


    /**
     * POST 请求，执行远程
     *
     * @param url
     * @param jsonStr
     * @param not200ThrowError
     * @return
     * @throws IOException
     */
    public static Map<String, Object> post(String url, String jsonStr, boolean not200ThrowError) throws IOException {
        return post(url, jsonStr, new HashMap<>(), not200ThrowError);
    }


    /**
     * POST 请求执行远程链接
     *
     * @param url
     * @param jsonStr 请求 Body 体
     * @param header 请求头
     * @param not200ThrowError
     * @return
     * @throws IOException
     */
    public static Map<String, Object> post(String url, String jsonStr, Map<String, String> header, boolean not200ThrowError) throws IOException {
        HttpPost post = new HttpPost(url);
        StringEntity entity = new StringEntity(jsonStr, "UTF-8");
        entity.setContentType("application/json");
        post.setEntity(entity);
        if(header != null && !header.isEmpty()) {
            for (Map.Entry<String, String> entry : header.entrySet()) {
                post.addHeader(entry.getKey(), entry.getValue());
            }
        }
        CloseableHttpResponse resp = null;
        try {
            resp = httpClient.execute(post);
            log.info("execute[post] url {} return code: {}", url, resp.getStatusLine().getStatusCode());
            HttpEntity entity1 = resp.getEntity();
            String result = EntityUtils.toString(entity1);
            EntityUtils.consume(entity1);
            if(not200ThrowError && resp.getStatusLine().getStatusCode() != 200) {
                throw new IOException(result);
            }
            Object jsonResult = JSON.parse(result);
            JSONObject jsonObject = new JSONObject(2);
            if(jsonResult instanceof JSONArray) {
                jsonObject.put("result", jsonResult);
            } else {
                jsonObject = (JSONObject) jsonResult;
            }
            jsonObject.put("status_code", resp.getStatusLine().getStatusCode());
            return jsonObject;
        } finally {
            if(resp != null) {
                resp.close();
            }
        }
    }

    /**
     * 执行 execute 返回 Map 数据对象
     * @param request
     * @param not200ThrowError
     * @return
     * @throws IOException
     */
    public static Map<String, Object> execute(HttpUriRequest request, boolean not200ThrowError) throws IOException {
        CloseableHttpResponse resp = null;
        try {
            resp = httpClient.execute(request);
            log.info("execute[{}] url {} return code: {}", request.getMethod(), request.getURI(), resp.getStatusLine().getStatusCode());
            HttpEntity entity1 = resp.getEntity();
            String result = EntityUtils.toString(entity1);
            EntityUtils.consume(entity1);
            if(not200ThrowError && resp.getStatusLine().getStatusCode() != 200) {
                throw new IOException(result);
            }
            Object jsonResult = JSON.parse(result);
            JSONObject jsonObject = new JSONObject(2);
            if(jsonResult instanceof JSONArray) {
                jsonObject.put("result", jsonResult);
            } else {
                jsonObject = (JSONObject) jsonResult;
            }
            jsonObject.put("status_code", resp.getStatusLine().getStatusCode());
            return jsonObject;
        } finally {
            if(resp != null) {
                resp.close();
            }
        }
    }

    /**
     * 执行任意 request，并采用回调 consumer
     * @param request
     * @param consumer
     * @throws IOException
     */
    public static void execute(HttpUriRequest request, Consumer<HttpResponse> consumer) throws IOException {
        CloseableHttpResponse resp = null;
        try {
            resp = httpClient.execute(request);
            log.info("execute[{}] url {} return code: {}", request.getMethod(), request.getURI(), resp.getStatusLine().getStatusCode());
            consumer.accept(resp);
        } finally {
            if (resp != null) {
                resp.close();
            }
        }
    }


    //---------
    /**
     * 调用一次远程 api
     *
     * @param url
     * @return
     * @throws IOException
     */
    public static Map<String, Object> get(String url) throws IOException {
        return get(url, new HashMap<>(), false);
    }


    /**
     * 调用一次远程 api
     *
     * @param url
     * @return
     * @throws IOException
     */
    public static Map<String, Object> get(String url, boolean not200ThrowError) throws IOException {
        return get(url, new HashMap<>(), not200ThrowError);
    }


    /**
     * 通过 Post 请求调用Rest 接口
     *
     * @param url
     * @param json
     * @return
     * @throws IOException
     */
    public static Map<String, Object> get(String url, Map<String, Object> json) throws IOException {
        return get(url, json, false);
    }

    /**
     * 通过 Post 请求调用Rest 接口
     *
     * @param url
     * @param json
     * @param not200ThrowError 为 true 时，当返回不是 200，则抛出异常
     * @return
     * @throws IOException
     */
    public static Map<String, Object> get(String url, Map<String, Object> json, boolean not200ThrowError) throws IOException {
        return get(url, json, new HashMap<>(), not200ThrowError);
    }

    /**
     * 通过 Post 请求调用Rest 接口
     *
     * @param url
     * @param json
     * @param not200ThrowError 为 true 时，当返回不是 200，则抛出异常
     * @return
     * @throws IOException
     */
    public static Map<String, Object> get(String url, Map<String, Object> json, Map<String, String> header, boolean not200ThrowError) throws IOException {
        HttpGet get = new HttpGet(url);
        if(json != null && !json.isEmpty()) {
            BasicHttpParams params = new BasicHttpParams();
            for (Map.Entry<String, Object> entry : json.entrySet()) {
                params.setParameter(entry.getKey(), entry.getValue());
            }
            get.setParams(params);
        }
        if(header != null && !header.isEmpty()) {
            for (Map.Entry<String, String> entry : header.entrySet()) {
                get.addHeader(entry.getKey(), entry.getValue());
            }
        }
        CloseableHttpResponse resp = null;
        try {
            resp = httpClient.execute(get);
            log.info("execute[get] url {} return code: {}", url, resp.getStatusLine().getStatusCode());
            final String contentType = Optional.ofNullable(resp.getFirstHeader("Content-Type")).map(Header::getValue).orElse("text/html");
            final HttpEntity entity = resp.getEntity();
            String result = EntityUtils.toString(entity, StandardCharsets.UTF_8);
            EntityUtils.consume(entity);
            if(not200ThrowError && resp.getStatusLine().getStatusCode() != 200) {
                throw new IOException(result);
            }
            // 判断一下返回 类型
            JSONObject jsonObject = new JSONObject(2);
            if(!contentType.contains("json")) {
                jsonObject.put("result", result);
            } else {
                Object jsonResult = JSON.parse(result);
            if(jsonResult instanceof JSONArray) {
                jsonObject.put("result", jsonResult);
            } else {
                jsonObject = (JSONObject) jsonResult;
            }
            }
            jsonObject.put("status_code", resp.getStatusLine().getStatusCode());
            return jsonObject;
        } finally {
            if(resp != null) {
                resp.close();
            }
        }
    }


    /**
     * 根据url下载文件，保存到filepath中
     *
     * @param url
     * @param downloadDir
     * @return 返回 下载的文件路径
     */
    public static String download(String url, File downloadDir) throws IOException {
        return download(url, new HashMap<>(), downloadDir, new NoProcessCall());
    }

    /**
     * 根据url下载文件，保存到filepath中
     *
     * @param url
     * @param downloadDir
     * @return 返回 下载的文件路径
     */
    public static String download(String url, Map<String, String> headers, File downloadDir) throws IOException {
        return download(url, headers, downloadDir, new NoProcessCall());
    }

    /**
     * 根据url下载文件，保存到filepath中
     *
     * @param url
     * @param downloadDir
     * @return 返回 下载的文件路径
     */
    public static String download(String url, Map<String, String> headers, File downloadDir, ProcessCall call) throws IOException {
        if(!downloadDir.exists()) {
            if(!downloadDir.mkdirs()) {
                throw new IOException(downloadDir.getAbsolutePath() + " not exists, do can not mkdir.");
            }
        }
        // 构造 Header，并且绑定，下载时登录，其实只要绑定 SessionID 就可以了
        HttpGet httpget = new HttpGet(url);
        if(headers != null) {
            for (Map.Entry<String, String> entry : headers.entrySet()) {
                httpget.setHeader(entry.getKey(), entry.getValue());
            }
        }

        // 开始下载
        HttpResponse response = httpClient.execute(httpget);
        String fileName = null;
        final Header contentType = response.getFirstHeader(HttpHeaders.CONTENT_TYPE);
        if(StringUtils.contains(contentType.getValue(), "application/octet-stream") ||
                StringUtils.contains(contentType.getValue(), "application/force-download")) {
            // 下载文件
            fileName = getFileName(response);
            if(StringUtils.isBlank(fileName)) {
                log.warn(response.getFirstHeader(HttpHeaders.CONTENT_DISPOSITION) + " can 'not found filename.");
            }
        }
        if(StringUtils.isBlank(fileName)) {
            fileName = getFileName(response);
        }
        if(StringUtils.isBlank(fileName)) {
            //无法从 header中获得文件名，如果路径是 /path/for/bar.zip  以 bar.zip 为文件名
            final String rawPath = httpget.getURI().getRawPath();
            if(!rawPath.endsWith("/")) {
                fileName = Paths.get(rawPath).getFileName().toString();
            }
            if(StringUtils.isBlank(fileName)) {
                log.warn("can not found download filename, use system timestamp.");
                fileName = String.valueOf(System.currentTimeMillis());
            }
        }
        log.info("download filename: {}", fileName);
        HttpEntity entity = response.getEntity();
        File filepath = new File(downloadDir, fileName);

        final long fileSize = entity.getContentLength();
        call.fileSize(fileSize);
        call.process(0.0f);
        try(InputStream is = entity.getContent(); FileOutputStream fileout = new FileOutputStream(filepath);) {
            byte[] buf = new byte[ 4 * 1024]; //4KB 缓冲区
            long total = 0;
            while (true) {
                int r = is.read(buf);
                if (r == -1) {
                    break;
                }
                fileout.write(buf, 0, r);
                total += r;
                if(total % 2048 == 1024) {
                    // 每 4M 推送一次进度
                    call.process(fileSize > 0 ? (float) total / fileSize : 0.0f);
                }
            }
            fileout.flush();
        }
        call.process(1.0f);
        return filepath.getAbsolutePath();
    }


    /**
     * 获取response header中Content-Disposition中的filename值
     *
     * @param response
     * @return
     */
    private static String getFileName(HttpResponse response) {
        Header contentHeader = response.getFirstHeader(HttpHeaders.CONTENT_DISPOSITION);
        String filename = null;
        if (contentHeader != null) {
            HeaderElement[] values = contentHeader.getElements();
            if (values.length >= 1) {
                NameValuePair param = values[0].getParameterByName("filename");
                if (param != null) {
                    try {
                        if(param.getValue() != null && param.getValue().contains("%")) {
                            //filename = new String(param.getValue().toString().getBytes(), "utf-8");
                            filename = URLDecoder.decode(param.getValue(), "UTF-8");
                        } else {
                            filename = param.getValue();
                        }
                    } catch (Exception e) {
                        filename = param.getValue();
                    }
                }
            }
        }
        return filename;
    }

    // -------------------------------上传文件-----------------------------------
    /**
     * 根据url上传文件
     *
     * @param url
     * @param uploadFile
     * @return 返回 下载的文件路径
     */
    public static Map<String, Object> upload(String url,
                                             File uploadFile) throws IOException {
        return upload(url, null, null, uploadFile);
    }

    /**
     * 根据url上传文件
     *
     * @param url
     * @param json payload 负载的消息参数
     * @param uploadFile
     * @return 返回 下载的文件路径
     */
    public static Map<String, Object> upload(String url,
                                             Map<String, Object> json,
                                             File uploadFile) throws IOException {
        return upload(url, json, null, uploadFile);
    }

    /**
     * 根据url上传文件
     *
     * @param url
     * @param json payload 负载的消息参数
     * @param headers Http Header
     * @param uploadFile
     * @return 返回 下载的文件路径
     */
    public static Map<String, Object> upload(String url,
                                             Map<String, Object> json,
                                             Map<String, String> headers,
                                             File uploadFile) throws IOException {
        return upload(url, json, headers, uploadFile, -1, -1, true);
    }

    /**
     * 根据url上传文件
     *
     * @param url
     * @param json payload 负载的消息参数
     * @param headers Http Header
     * @param uploadFile
     * @param usePut 采用 PUT 请求上传文件
     * @return 返回 下载的文件路径
     */
    public static Map<String, Object> upload(String url,
                                             Map<String, Object> json,
                                             Map<String, String> headers,
                                             File uploadFile,
                                             boolean usePut) throws IOException {
        return upload(url, json, headers, uploadFile, -1, -1, usePut, true);
    }

    /**
     * 根据 url 上传文件
     *
     * @param url
     * @param json payload 负载的消息
     * @param headers Http Header
     * @param uploadFile
     * @return 返回 下载的文件路径
     */
    public static Map<String, Object> upload(String url,
                                             Map<String, Object> json,
                                             Map<String, String> headers,
                                             File uploadFile,
                                             int connectTimeout,
                                             int socketTimeout,
                                             boolean not200ThrowError) throws IOException {
        return upload(url, json, headers, uploadFile, connectTimeout, socketTimeout, false, not200ThrowError);
    }

    /**
     * 根据 url 上传文件
     *
     * @param url
     * @param json payload 负载的消息
     * @param headers Http Header
     * @param uploadFile
     * @param usePut 采用 PUT 请求
     * @return 返回 下载的文件路径
     */
    public static Map<String, Object> upload(String url,
                                             Map<String, Object> json,
                                             Map<String, String> headers,
                                             File uploadFile,
                                             int connectTimeout,
                                             int socketTimeout,
                                             boolean usePut,
                                             boolean not200ThrowError) throws IOException {
        // 文件夹 或者 是 文件不存在
        if(!uploadFile.exists()) {
            throw new FileNotFoundException("upload file: " + uploadFile.getAbsolutePath() + " can not found!");
        }
        if(uploadFile.isDirectory()) {
            throw new IllegalStateException("upload payload must be file, but found directory.");
        }

        HttpEntityEnclosingRequestBase httpPost = usePut ? new HttpPut(url) : new HttpPost(url);
        if(connectTimeout > 0 || socketTimeout >= 0) {
            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(connectTimeout <= 0 ? 30000 : connectTimeout)  // 服务器端链接超时设置 30s
                    .setSocketTimeout(socketTimeout <= 0 ? 10 * 60 * 1000 : socketTimeout)  // 上传等待 10 分钟
                    .build();
            httpPost.setConfig(requestConfig);
        }
        if(headers != null) {
            for (Map.Entry<String, String> entry : headers.entrySet()) {
                httpPost.setHeader(entry.getKey(), entry.getValue());
            }
        }

        if(json != null && !json.isEmpty()) {
            // 有文件上传，并且又负载的消息
            MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create();
            multipartEntityBuilder.setCharset(StandardCharsets.UTF_8);

            //multipartEntityBuilder.addBinaryBody("file", file,ContentType.create("image/png"),"abc.pdf");
            //当设置了setSocketTimeout参数后，以下代码上传PDF不能成功，将setSocketTimeout参数去掉后此可以上传成功。上传图片则没有个限制
            //multipartEntityBuilder.addBinaryBody("file",file,ContentType.create("application/octet-stream"),"abd.pdf");
            multipartEntityBuilder.addBinaryBody("file", uploadFile);
            for (Map.Entry<String, Object> entry : json.entrySet()) {
                multipartEntityBuilder.addTextBody(entry.getKey(), (String) entry.getValue());
            }
            HttpEntity httpEntity = multipartEntityBuilder.build();
            httpPost.setEntity(httpEntity);
        } else {
            // 只有文件上传
            FileEntity entity = new FileEntity(uploadFile);
            httpPost.setEntity(entity);
        }
        try (CloseableHttpResponse httpResponse = httpClient.execute(httpPost);){
            HttpEntity responseEntity = httpResponse.getEntity();
            int statusCode = httpResponse.getStatusLine().getStatusCode();
            String result = EntityUtils.toString(responseEntity);
            EntityUtils.consume(responseEntity);
            if (not200ThrowError && httpResponse.getStatusLine().getStatusCode() != 200) {
                throw new IOException(result);
            }
            Object jsonResult = JSON.parse(result);
            JSONObject jsonObject = new JSONObject(2);
            if (jsonResult instanceof JSONArray) {
                jsonObject.put("result", jsonResult);
            } else {
                jsonObject = (JSONObject) jsonResult;
            }
            jsonObject.put("status_code", statusCode);
            return jsonObject;
        }
    }

    /**
     * 关闭 RPC 调用
     *
     * @throws IOException
     */
    public static void shutdown() throws IOException {
        httpClient.close();
    }

    /**
     * 下载进度回调
     */
    public static interface ProcessCall {

        /**
         * 下载的文件大小
         * @param size
         */
        default void fileSize(long size) {};

        /**
         * 进度推送
         * @param process
         */
        public void process(float process);
    }

    static class NoProcessCall implements ProcessCall {

        @Override
        public void process(float process) {

        }
    }
}