package org.springblade.common.utils;
|
|
import com.alibaba.fastjson.JSON;
|
import org.apache.http.HttpEntity;
|
import org.apache.http.NameValuePair;
|
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
import org.apache.http.client.methods.HttpGet;
|
import org.apache.http.client.methods.HttpPost;
|
import org.apache.http.client.utils.URIBuilder;
|
import org.apache.http.config.Registry;
|
import org.apache.http.config.RegistryBuilder;
|
import org.apache.http.conn.socket.ConnectionSocketFactory;
|
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
|
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
import org.apache.http.conn.ssl.TrustStrategy;
|
import org.apache.http.entity.StringEntity;
|
import org.apache.http.impl.client.*;
|
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
import org.apache.http.message.BasicNameValuePair;
|
import org.apache.http.ssl.SSLContextBuilder;
|
import org.apache.http.util.EntityUtils;
|
import sun.misc.BASE64Encoder;
|
|
import javax.net.ssl.SSLContext;
|
import java.io.BufferedReader;
|
import java.io.IOException;
|
import java.io.InputStreamReader;
|
import java.io.OutputStreamWriter;
|
import java.net.HttpURLConnection;
|
import java.net.URL;
|
import java.security.KeyManagementException;
|
import java.security.KeyStoreException;
|
import java.security.NoSuchAlgorithmException;
|
import java.security.cert.CertificateException;
|
import java.security.cert.X509Certificate;
|
import java.util.*;
|
|
|
public class HttpClientUtils {
|
/**
|
* 执行有参GET请求
|
*
|
* @param url
|
* @param params
|
* @return
|
*/
|
public static String doGet(String url, Map<String, String> params) {
|
|
//获取httpclient客户端
|
CloseableHttpClient httpclient = HttpClients.createDefault();
|
|
String resultString = "";
|
|
CloseableHttpResponse response = null;
|
|
try {
|
URIBuilder builder = new URIBuilder(url);
|
|
if (null != params) {
|
for (String key : params.keySet()) {
|
builder.setParameter(key, params.get(key));
|
}
|
}
|
|
HttpGet get = new HttpGet(builder.build());
|
|
response = httpclient.execute(get);
|
|
System.out.println(response.getStatusLine());
|
|
if (200 == response.getStatusLine().getStatusCode()) {
|
HttpEntity entity = response.getEntity();
|
resultString = EntityUtils.toString(entity, "utf-8");
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
} finally {
|
if (null != response) {
|
try {
|
response.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
if (null != httpclient) {
|
try {
|
httpclient.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
}
|
|
return resultString;
|
}
|
|
/**
|
* 执行有参GET请求,带请求头
|
*
|
* @param url 请求url
|
* @param params 参数
|
* @param key 请求头Key
|
* @param secretKey 秘钥
|
* @return
|
*/
|
public static String doGetHeader(String url, String key, String secretKey, Map<String, String> params) {
|
|
//获取httpclient客户端
|
CloseableHttpClient httpclient = HttpClients.createDefault();
|
|
String resultString = "";
|
|
CloseableHttpResponse response = null;
|
|
try {
|
URIBuilder builder = new URIBuilder(url);
|
|
if (null != params) {
|
for (String keys : params.keySet()) {
|
builder.addParameter(keys, params.get(keys));
|
//builder.setParameter(keys, params.get(keys));
|
}
|
}
|
|
HttpGet httpGet = new HttpGet(builder.build());
|
|
//设置请求头
|
httpGet.addHeader(key, secretKey);
|
// 传输的类型
|
httpGet.addHeader("Content-Type", "application/x-www-form-urlencoded");
|
//执行Http请求调用
|
response = httpclient.execute(httpGet);
|
//判断是否请求成功返回
|
if (200 == response.getStatusLine().getStatusCode()) {
|
HttpEntity entity = response.getEntity();
|
resultString = EntityUtils.toString(entity, "utf-8");
|
}
|
|
} catch (Exception e) {
|
e.printStackTrace();
|
} finally {
|
if (null != response) {
|
try {
|
response.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
if (null != httpclient) {
|
try {
|
httpclient.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
}
|
|
return resultString;
|
}
|
|
/**
|
* 执行有参GET请求,带请求头,接收图片流
|
*
|
* @param url 请求url
|
* @param params 参数
|
* @param key 请求头Key
|
* @param secretKey 秘钥
|
* @return
|
*/
|
public static String doGetHeaderPictureBase64(String url, String key, String secretKey, Map<String, String> params) {
|
|
//获取httpclient客户端
|
CloseableHttpClient httpclient = HttpClients.createDefault();
|
String resultString = "";
|
CloseableHttpResponse response = null;
|
try {
|
URIBuilder builder = new URIBuilder(url);
|
if (null != params) {
|
for (String keys : params.keySet()) {
|
builder.addParameter(keys, params.get(keys));
|
}
|
}
|
HttpGet httpGet = new HttpGet(builder.build());
|
//设置请求头
|
httpGet.addHeader(key, secretKey);
|
// 传输的类型
|
httpGet.addHeader("Content-Type", "application/x-www-form-urlencoded");
|
//执行Http请求调用
|
response = httpclient.execute(httpGet);
|
// 将返回的图片或者文件转化成字节数组的形式
|
byte[] data = EntityUtils.toByteArray(response.getEntity());
|
BASE64Encoder encoder = new BASE64Encoder();
|
//String imageBase64 = "data:image/png;base64," + encoder.encodeBuffer(data).trim();
|
return encoder.encodeBuffer(data).trim().replaceAll("\n", "").replaceAll("\r", "").replaceAll(" ", "");//删除 \r\n
|
} catch (Exception e) {
|
e.printStackTrace();
|
} finally {
|
if (null != response) {
|
try {
|
response.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
if (null != httpclient) {
|
try {
|
httpclient.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
}
|
return resultString;
|
}
|
|
/**
|
* 执行无参GET请求
|
*
|
* @param url
|
* @return
|
*/
|
public static String doGet(String url) {
|
return doGet(url, null);
|
}
|
|
/**
|
* 执行有参POST请求
|
*
|
* @param url
|
* @param params
|
* @return
|
*/
|
public static String doPost(String url, Map<String, String> params) {
|
/**
|
* 在4.0及以上httpclient版本中，post需要指定重定向的策略，如果不指定则按默认的重定向策略。
|
*
|
* 获取httpclient客户端
|
*/
|
CloseableHttpClient httpclient = HttpClientBuilder.create().setRedirectStrategy(new LaxRedirectStrategy()).build();
|
|
String resultString = "";
|
|
CloseableHttpResponse response = null;
|
|
try {
|
|
|
HttpPost post = new HttpPost(url);
|
|
List<NameValuePair> paramaters = new ArrayList<>();
|
|
if (null != params) {
|
for (String key : params.keySet()) {
|
paramaters.add(new BasicNameValuePair(key, params.get(key)));
|
}
|
// 构造一个form表单式的实体
|
UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(paramaters);
|
// 将请求实体设置到httpPost对象中
|
post.setEntity(formEntity);
|
}
|
|
|
/**
|
* HTTP/1.1 403 Forbidden
|
* 原因：
|
* 有些网站，设置了反爬虫机制
|
* 解决的办法：
|
* 设置请求头，伪装浏览器
|
*/
|
post.addHeader("user-agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
|
|
response = httpclient.execute(post);
|
|
System.out.println(response.getStatusLine());
|
|
if (200 == response.getStatusLine().getStatusCode()) {
|
HttpEntity entity = response.getEntity();
|
resultString = EntityUtils.toString(entity, "utf-8");
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
} finally {
|
if (null != response) {
|
try {
|
response.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
if (null != httpclient) {
|
try {
|
httpclient.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
}
|
return resultString;
|
}
|
|
public static String doPost(String url) {
|
return doPost(url, null);
|
}
|
|
|
public static void main(String[] args) {
|
|
Map<String, String> params = new HashMap<>();
|
params.put("scope", "project");
|
params.put("q", "数据库");
|
//
|
// System.out.println(doGet("http://www.baidu.com/s",params));
|
|
/**
|
* 有一部分网站，禁止爬虫技术访问网站。
|
*
|
* 解决方案：
|
* 伪装浏览器
|
* post.addHeader("user-agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");
|
*/
|
System.out.println(doPost("http://www.oschina.net/search", params));
|
}
|
|
/**
|
* post 请求 header 带 秘钥
|
*
|
* @param url
|
* @param appKey
|
* @param appKeyValue
|
* @param map
|
* @return
|
*/
|
public static String httpPost(String url, String appKey, String appKeyValue, Map<String, String> map) {
|
// 返回body
|
String body = null;
|
// 获取连接客户端工具
|
CloseableHttpClient httpClient = HttpClients.createDefault();
|
CloseableHttpResponse httpResponse = null;
|
// 2、创建一个HttpPost请求
|
HttpPost post = new HttpPost(url);
|
// 5、设置header信息
|
/**header中通用属性*/
|
// post.setHeader("Accept", "*/*");
|
// post.setHeader("Accept-Encoding", "gzip, deflate");
|
// post.setHeader("Cache-Control", "no-cache");
|
// post.setHeader("Connection", "keep-alive");
|
post.setHeader("Content-Type", "application/json;charset=UTF-8");
|
/**业务参数*/
|
post.setHeader(appKey, appKeyValue);
|
// 设置参数
|
if (map != null) {
|
try {
|
StringEntity entity1 = new StringEntity(JSON.toJSONString(map), "UTF-8");
|
entity1.setContentEncoding("UTF-8");
|
entity1.setContentType("json/form-data");
|
post.setEntity(entity1);
|
|
// 7、执行post请求操作，并拿到结果
|
httpResponse = httpClient.execute(post);
|
// 获取结果实体
|
HttpEntity entity = httpResponse.getEntity();
|
if (entity != null) {
|
// 按指定编码转换结果实体为String类型
|
body = EntityUtils.toString(entity, "UTF-8");
|
}
|
try {
|
httpResponse.close();
|
httpClient.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
}
|
return body;
|
}
|
|
public static SSLContext createIgnoreVerifySSL() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException {
|
|
SSLContext sc = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
|
|
public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
|
return true;
|
}
|
|
}).build();
|
|
return sc;
|
}
|
|
//适用于post请求并传送form-data数据（同样适用于post的Raw类型的application-json格式）
|
public static String postParams(String url, String appKey, Map<String, String> map) {
|
SSLContext sslcontext = null;
|
try {
|
sslcontext = createIgnoreVerifySSL();
|
} catch (KeyManagementException e) {
|
e.printStackTrace();
|
} catch (NoSuchAlgorithmException e) {
|
e.printStackTrace();
|
} catch (KeyStoreException e) {
|
e.printStackTrace();
|
}
|
// 设置协议http和https对应的处理socket链接工厂的对象
|
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
|
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
.register("https", new SSLConnectionSocketFactory(sslcontext))
|
.build();
|
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
|
|
//创建自定义的httpclient对象
|
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).build();
|
HttpPost post = new HttpPost(url);
|
post.setHeader("appKey", appKey);
|
CloseableHttpResponse res = null;
|
try {
|
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
|
Set<String> keySet = map.keySet();
|
for (String key : keySet) {
|
nvps.add(new BasicNameValuePair(key, map.get(key)));
|
}
|
post.setEntity(new UrlEncodedFormEntity(nvps, "utf-8"));
|
res = client.execute(post);
|
HttpEntity entity = res.getEntity();
|
return EntityUtils.toString(entity, "utf-8");
|
} catch (Exception e) {
|
e.printStackTrace();
|
} finally {
|
try {
|
res.close();
|
client.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
return "";
|
}
|
|
|
//适用于post请求并传送form-data数据（同样适用于post的Raw类型的application-json格式）
|
public static String postParams(String url, Map<String, Object> map) {
|
SSLContext sslcontext = null;
|
try {
|
sslcontext = createIgnoreVerifySSL();
|
} catch (KeyManagementException e) {
|
e.printStackTrace();
|
} catch (NoSuchAlgorithmException e) {
|
e.printStackTrace();
|
} catch (KeyStoreException e) {
|
e.printStackTrace();
|
}
|
// 设置协议http和https对应的处理socket链接工厂的对象
|
Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
|
.register("http", PlainConnectionSocketFactory.INSTANCE)
|
.register("https", new SSLConnectionSocketFactory(sslcontext))
|
.build();
|
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
|
|
//创建自定义的httpclient对象
|
CloseableHttpClient client = HttpClients.custom().setConnectionManager(connManager).build();
|
HttpPost post = new HttpPost(url);
|
// post.setHeader(appKey, appKeyValue);
|
CloseableHttpResponse res = null;
|
try {
|
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
|
Set<String> keySet = map.keySet();
|
for (String key : keySet) {
|
nvps.add(new BasicNameValuePair(key, map.get(key).toString()));
|
}
|
post.setEntity(new UrlEncodedFormEntity(nvps, "utf-8"));
|
res = client.execute(post);
|
HttpEntity entity = res.getEntity();
|
return EntityUtils.toString(entity, "utf-8");
|
} catch (Exception e) {
|
e.printStackTrace();
|
} finally {
|
try {
|
res.close();
|
client.close();
|
} catch (IOException e) {
|
e.printStackTrace();
|
}
|
}
|
return "";
|
}
|
|
/**
|
* 向指定 URL 发送POST方法的请求
|
*
|
* @param url 发送请求的 URL
|
* @param params 请求的参数集合
|
* @return 远程资源的响应结果
|
*/
|
@SuppressWarnings("unused")
|
public static String sendPost(String url, Map<String, String> params) {
|
OutputStreamWriter out = null;
|
BufferedReader in = null;
|
StringBuilder result = new StringBuilder();
|
try {
|
URL realUrl = new URL(url);
|
HttpURLConnection conn = (HttpURLConnection) realUrl.openConnection();
|
// 发送POST请求必须设置如下两行
|
conn.setDoOutput(true);
|
conn.setDoInput(true);
|
// POST方法
|
conn.setRequestMethod("POST");
|
// 设置通用的请求属性
|
conn.setRequestProperty("accept", "*/*");
|
conn.setRequestProperty("connection", "Keep-Alive");
|
conn.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
|
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
|
conn.connect();
|
// 获取URLConnection对象对应的输出流
|
out = new OutputStreamWriter(conn.getOutputStream(), "UTF-8");
|
// 发送请求参数
|
if (params != null) {
|
StringBuilder param = new StringBuilder();
|
for (Map.Entry<String, String> entry : params.entrySet()) {
|
if (param.length() > 0) {
|
param.append("&");
|
}
|
param.append(entry.getKey());
|
param.append("=");
|
param.append(entry.getValue());
|
// System.out.println(entry.getKey()+":"+entry.getValue());
|
}
|
// System.out.println("param:"+param.toString());
|
out.write(param.toString());
|
}
|
// flush输出流的缓冲
|
out.flush();
|
// 定义BufferedReader输入流来读取URL的响应
|
in = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
|
String line;
|
while ((line = in.readLine()) != null) {
|
result.append(line);
|
}
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
// 使用finally块来关闭输出流、输入流
|
finally {
|
try {
|
if (out != null) {
|
out.close();
|
}
|
if (in != null) {
|
in.close();
|
}
|
} catch (IOException ex) {
|
ex.printStackTrace();
|
}
|
}
|
return result.toString();
|
}
|
|
|
/**
|
* json body
|
*
|
* @param url
|
* @param json
|
* @return
|
* @throws IOException
|
*/
|
public static String httpPostWithjson(String url, String json) throws IOException {
|
String result = "";
|
HttpPost httpPost = new HttpPost(url);
|
CloseableHttpClient httpClient = HttpClients.createDefault();
|
try {
|
BasicResponseHandler handler = new BasicResponseHandler();
|
//解决中文乱码问题
|
StringEntity entity = new StringEntity(json, "utf-8");
|
entity.setContentEncoding("UTF-8");
|
entity.setContentType("application/json");
|
httpPost.setEntity(entity);
|
result = httpClient.execute(httpPost, handler);
|
return result;
|
} catch (Exception e) {
|
e.printStackTrace();
|
} finally {
|
try {
|
httpClient.close();
|
} catch (Exception e) {
|
e.printStackTrace();
|
}
|
}
|
return result;
|
}
|
}
|
