最近GFW为了刷存在感，搞得大家是头晕眼花，修改hosts 几乎成了每日必备工作。
索性写了一个小程序，给办公室的同事们分享，其中有个内容 就是抓取网络上的hosts，废了一些周折。
我是在一个博客上抓取的。但是这位朋友的博客应该是在做防盗链，但他的方式比较简单就是5位数的一个整形随机数。这里折腾一下就ok了。
要命的是他这个链接的流类型 居然是gzip。这个郁闷好久，一直以为是编码格式导致解析不出来结果，后来发现是gzip搞的。
主要的一段代码做个记录吧。
1 /**
2 * 网络工具类 用于抓取http://serve.netsh.org上的hosts数据
3 *
4 * @author tone
5 */
6 public class NetUtil {
7
8 private final static String ENCODING = "UTF-8";
9 private final static String GZIPCODING = "gzip";
10 private final static String HOST = "http://serve.netsh.org/pub/hosts.php";
11 private final static String COOKIE = "hostspasscode=%s; Hm_lvt_e26a7cd6079c926259ded8f19369bf0b=1421846509,1421846927,1421847015,1421849633; Hm_lpvt_e26a7cd6079c926259ded8f19369bf0b=1421849633";
12 private final static String OFF = "off";
13 private final static String ON = "on";
14 private final static int RANDOM = 100000;
15 private static String hostspasscode = null;
16 private static NetUtil instance;
17
18 public static NetUtil getInstance() {
19 if (instance == null) {
20 instance = new NetUtil();
21 }
22 return instance;
23 }
24
25 private NetUtil() {
26 hostspasscode = createRandomCookies();
27 }
28
29 /**
30 * 获取html内容
31 *
32 * @param gs
33 * @param wk
34 * @param twttr
35 * @param fb
36 * @param flkr
37 * @param dpbx
38 * @param odrvB
39 * @param yt
40 * @param nohl
41 * @return
42 */
43 public String getHtmlInfo(boolean gs, boolean wk, boolean twttr, boolean fb,
44 boolean flkr, boolean dpbx, boolean odrv,
45 boolean yt, boolean nohl) throws Exception {
46 HttpURLConnection conn = null;
47
48 String result = "";
49
50 //String cookie = "hostspasscode="+hostspasscode+"; Hm_lvt_e26a7cd6079c926259ded8f19369bf0b=1421846509,1421846927,1421847015,1421849633; Hm_lpvt_e26a7cd6079c926259ded8f19369bf0b=1421849633";
51 String cookie = String.format(COOKIE, hostspasscode);
52
53 //URL url = new URL("http://serve.netsh.org/pub/hosts.php?passcode=13008&gs=on&wk=on&twttr=on&fb=on&flkr=on&dpbx=on&odrv=on&yt=on&nolh=on");
54 URL url = new URL(createUrl(hostspasscode, gs, wk, twttr, fb, flkr, dpbx, odrv, yt, nohl));
55 //System.out.println(cookie);
56 // System.out.println(url.toString());
57
58 conn = (HttpURLConnection) url.openConnection();
59
60 conn.setConnectTimeout(5 * 1000);
61 conn.setDoOutput(true);
62 //get方式提交
63 conn.setRequestMethod("GET");
64 //凭借请求头文件
65 conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
66 conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
67 conn.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
68 conn.setRequestProperty("Connection", "keep-alive");
69 conn.setRequestProperty("Cookie", cookie);
70 conn.setRequestProperty("Host", "serve.netsh.org");
71 conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0");
72
73 // conn.setRequestProperty("Referer", "http://serve.netsh.org/pub/gethosts.php");
74 // conn.setRequestProperty("X-Requested-With", "XMLHttpRequest");
75
76 conn.connect();
77
78 String encoding = conn.getContentEncoding();
79
80 result = readStream(conn.getInputStream(), encoding);
81 //测试进度条显示
82 // result = readStream(new FileInputStream(new File("/home/tone/Resident.Evil.Damnation.2012.1080p.BluRay.x264.DTS-WiKi.mkv")), "11");
83
84 conn.disconnect();
85 if (nohl) {
86 result=getLocalHost()+result;
87 }
88
89 return result;
90 }
91
92 /**
93 * 读取将InputStream中的字节读以字符的形式取到字符串中，如果encoding是gzip，那么需要先有GZIPInputStream进行封装
94 *
95 * @param inputStream InputStream字节流
96 * @param encoding 编码格式
97 * @return String类型的形式
98 * @throws IOException IO异常
99 */
100 private String readStream(InputStream inputStream, String encoding) throws Exception {
101 StringBuffer buffer = new StringBuffer();
102 ProgressMonitorInputStream pmis = null;
103
104 InputStreamReader inputStreamReader = null;
105 GZIPInputStream gZIPInputStream = null;
106 if (GZIPCODING.equals(encoding)) {
107 gZIPInputStream = new GZIPInputStream(inputStream);
108 inputStreamReader = new InputStreamReader(ProgressUtil.getMonitorInputStream(gZIPInputStream, "获取网络数据"), ENCODING);
109
110 } else {
111
112 inputStreamReader = new InputStreamReader(ProgressUtil.getMonitorInputStream(inputStream, "获取网络数据"), ENCODING);
113 }
114
115
116 char[] c = new char[1024];
117
118 int lenI;
119 while ((lenI = inputStreamReader.read(c)) != -1) {
120
121 buffer.append(new String(c, 0, lenI));
122
123 }
124 if (inputStream != null) {
125 inputStream.close();
126 }
127 if (gZIPInputStream != null) {
128 gZIPInputStream.close();
129 }
130 if (pmis!=null) {
131 gZIPInputStream.close();
132 }
133
134
135 return buffer.toString();
136
137
138 }
139
140 /**
141 * 生成随机Cookies数组
142 *
143 * @return 五位随机数字
144 */
145 private String createRandomCookies() {
146
147 return String.valueOf(Math.random() * RANDOM).substring(0, 5);
148
149 }
150
151 /**
152 * 生成链接字符串
153 *
154 * @param hostspasscode
155 * @param gs
156 * @param wk
157 * @param twttr
158 * @param fb
159 * @param flkr
160 * @param dpbx
161 * @param odrvB
162 * @param yt
163 * @param nohl
164 * @return
165 */
166 private String createUrl(String hostspasscode, boolean gs, boolean wk, boolean twttr, boolean fb,
167 boolean flkr, boolean dpbx, boolean odrv,
168 boolean yt, boolean nohl) {
169 StringBuffer buffer = new StringBuffer();
170 buffer.append(HOST);
171 buffer.append("?passcode=" + hostspasscode);
172 if (gs) {
173 buffer.append("&gs=" + ON);
174 } else {
175 buffer.append("&gs=" + OFF);
176 }
177 if (wk) {
178 buffer.append("&wk=" + ON);
179 } else {
180 buffer.append("&wk=" + OFF);
181 }
182 if (twttr) {
183 buffer.append("&twttr=" + ON);
184 } else {
185 buffer.append("&twttr=" + OFF);
186 }
187 if (fb) {
188 buffer.append("&fb=" + ON);
189 } else {
190 buffer.append("&fb=" + OFF);
191 }
192 if (flkr) {
193 buffer.append("&flkr=" + ON);
194 } else {
195 buffer.append("&flkr=" + OFF);
196 }
197 if (dpbx) {
198 buffer.append("&dpbx=" + ON);
199 } else {
200 buffer.append("&dpbx=" + OFF);
201 }
202 if (odrv) {
203 buffer.append("&odrv=" + ON);
204 } else {
205 buffer.append("&odrv=" + OFF);
206 }
207 if (yt) {
208 buffer.append("&yt=" + ON);
209 } else {
210 buffer.append("&yt=" + OFF);
211 }
212 if (nohl) {
213 buffer.append("&nohl=" + ON);
214 } else {
215 buffer.append("&nohl=" + OFF);
216 }
217 return buffer.toString();
218 }
219
220 private String getLocalHost() throws Exception {
221
222 StringBuffer buffer=new StringBuffer();
223 String hostName=OSUtil.getInstance().getLocalhostName();
224 buffer.append("#LOCALHOST begin"+"\n");
225 buffer.append("127.0.0.1\tlocalhost"+"\n");
226 if (hostName!=null&&!"".equals(hostName)) {
227 buffer.append("127.0.1.1\t"+hostName+"\n");
228 }
229
230 buffer.append("#LOCALHOST end"+"\n");
231 return buffer.toString();
232
233
234
235 }
236
237 }
时间： 2024-08-03 18:59:55