博客
关于我
强烈建议你试试无所不能的chatGPT,快点击我
基于apache的HttpComponents子项目抓取网页内容的HttpClientBrow...
阅读量:6375 次
发布时间:2019-06-23

本文共 4725 字,大约阅读时间需要 15 分钟。

  hot3.png

import java.io.IOException;import java.util.ArrayList;import java.util.HashMap;import org.apache.http.Header;import org.apache.http.HttpResponse;import org.apache.http.HttpStatus;import org.apache.http.NameValuePair;import org.apache.http.ProtocolVersion;import org.apache.http.client.ClientProtocolException;import org.apache.http.client.HttpClient;import org.apache.http.client.entity.UrlEncodedFormEntity;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.methods.HttpPost;import org.apache.http.impl.client.DefaultHttpClient;import org.apache.http.message.BasicNameValuePair;import org.apache.http.params.BasicHttpParams;import org.apache.http.params.HttpConnectionParams;import org.apache.http.params.HttpParams;import org.apache.http.params.HttpProtocolParams;import org.apache.http.util.EntityUtils;public class HttpClientBrowser {		private static final ProtocolVersion PROTOCOL_VERSION = new ProtocolVersion("HTTP", 1, 1);		public static int timeout=15000;		private HashMap
COOKIE = new HashMap
(); public void getcookie(HttpResponse conn) { String cookieVal = null; for(Header bfh:conn.getAllHeaders()){ if (bfh.getName().equalsIgnoreCase("set-cookie")) { cookieVal=bfh.getValue(); String[] keys = cookieVal.split(";"); for (int k = 0; k < keys.length; k++) { String[] vls = keys[k].split("="); if (",Path,Max-Age,Domain,Comment,".indexOf(vls[0])==-1&&vls.length>1) { COOKIE.put(vls[0], vls[1]); } } } } } public String cookie2String() { String retn = ""; for (String key : COOKIE.keySet()) { retn += key + "=" + COOKIE.get(key) + ";"; } return retn; } public String getHttpSessionId() { return COOKIE.get("JSESSIONID"); } private HttpResponse post(String url,ArrayList
list) throws ClientProtocolException, IOException,Exception{ HttpPost httpPost = new HttpPost(url); String cookie = cookie2String(); if (cookie.length() > 0) httpPost.setHeader("Cookie", cookie); HttpParams params = new BasicHttpParams(); HttpConnectionParams.setConnectionTimeout(params,timeout); HttpConnectionParams.setSoTimeout(params,timeout); if(list!=null){ httpPost.setEntity(new UrlEncodedFormEntity(list,"UTF-8")); } HttpProtocolParams.setVersion(params, PROTOCOL_VERSION); httpPost.setParams(params); HttpClient httpClient = new DefaultHttpClient(); HttpResponse response =httpClient.execute(httpPost); int code = response.getStatusLine().getStatusCode(); if(code!=HttpStatus.SC_OK){ System.out.println("HTTP状态返回码:"+code); System.out.println("HTTP返回内容:"+EntityUtils.toString(response.getEntity(), "gb2312")); throw new Exception("HTTP访问出错!"); } getcookie(response); return response; } public HttpResponse post(String url,String[][] params) throws ClientProtocolException, IOException,Exception{ ArrayList
list = new ArrayList
(); for(int i=0;i
=2){ list.add(new BasicNameValuePair(params[i][0],params[i][1])); } } return post(url,list); } public HttpResponse get(String url) throws ClientProtocolException, IOException,Exception{ HttpGet httpGet = new HttpGet(url); String cookie = cookie2String(); if (cookie.length() > 0) httpGet.setHeader("Cookie", cookie); HttpParams params = new BasicHttpParams(); HttpConnectionParams.setConnectionTimeout(params,timeout); HttpConnectionParams.setSoTimeout(params,timeout); HttpProtocolParams.setVersion(params, PROTOCOL_VERSION); httpGet.setParams(params); HttpClient httpClient = new DefaultHttpClient(); HttpResponse response =httpClient.execute(httpGet); int code = response.getStatusLine().getStatusCode(); if(code!=HttpStatus.SC_OK){ System.out.println("HTTP状态返回码:"+code); System.out.println("HTTP返回内容:"+EntityUtils.toString(response.getEntity(), "gb2312")); throw new Exception("HTTP访问出错!"); } getcookie(response); return response; } public String html(String url) throws ClientProtocolException, IOException,Exception{ HttpResponse res = get(url); return EntityUtils.toString(res.getEntity()); } public String html(String url,String encode) throws ClientProtocolException, IOException,Exception{ HttpResponse res = get(url); return EntityUtils.toString(res.getEntity(),encode); } public String html(String url,String[][] params) throws ClientProtocolException, IOException,Exception{ HttpResponse res = post(url,params); return EntityUtils.toString(res.getEntity()); } public String html(String url,String[][] params,String encode) throws ClientProtocolException, IOException,Exception{ HttpResponse res = post(url,params); return EntityUtils.toString(res.getEntity(),encode); } }//同一个HttpClientBrowser对象对于url的多次访问,保持同一个session访问。

转载于:https://my.oschina.net/hchen/blog/63966

你可能感兴趣的文章
图形化分析工具_推荐一款基于小米 soar 的开源 sql 分析与优化的 web 图形化工具...
查看>>
485 工控机 接线方式_RS485温湿度传感器的接线方式及注意事项
查看>>
12306排队是什么意思_12306订单请求排队中是什么意思 要等多久
查看>>
u3d游戏开发视频潭州_unity3d教程视频-unity3d教程中文零基础入门/u3d游戏开发/特效案例/项目实战V2.0 最新版 - 极光站...
查看>>
c++区块链实例_cpp 区块链模拟示例(二)工程代码解析
查看>>
hourglass论文_论文笔记 Stacked Hourglass Networks for Human Pose Estimation
查看>>
java 接口的本质_Java基本概念:接口
查看>>
java死锁的解决_java中常见的死锁以及解决方法代码
查看>>
java菜单栏不下拉_我java代码中的下拉列表设好后为什么无法下拉?
查看>>
java传递引用类型的实质_java的引用类型以及值传递
查看>>
java策略模式使用场景,Java设计模式—策略模式
查看>>
RHEL6.3实现基于加密的用户认证验证访问
查看>>
SCCM2012 R2实战系列之十一:解决OSD分发Windows7 系统盘盘符为’D’问题
查看>>
经验分享:我是如何在网店无货源情况下快速出单?
查看>>
当AD服务器置于防火墙内时,所需开放的端口
查看>>
限免的Mac App套件,工程师绝对不可错过
查看>>
Exchange 2013 添加地址列表到脱机通讯簿
查看>>
Skype for Business Server 2015-05-监控和存档服务器-配置
查看>>
浅谈物化视图
查看>>
安装SQL Server 2017
查看>>