简单爬虫 从指定地址下载网站内容

Http01App.java 1.使用了多线程、io流，net（网络包）

package main;import java.io.*;import java.net.HttpURLConnection;import java.net.MalformedURLException;import java.net.URL;import java.net.URLConnection;/** * Created by lxj-pc on 2017/6/27. */public class Http01App {    public static void main(String[] args) {        String url = "http://tuijian.hao123.com:80/index.html";        //启动线程下载指定位置的html内容        new Thread(new DownloadHtmlTask(url)).start();        //下载指定的html内容，并存在d://html/hao123.html    }

static class DownloadHtmlTask implements Runnable {        private String url;        String fileName = "hao123.html";        String dirPath = "d:/lxj";

public DownloadHtmlTask(String url) {

this.url = url;        }

@Override        public void run() {            //下载url指定的HTMl网页内容            try {                URL htmlURL = new URL(url);                //打开网络资源连接                try {                    URLConnection urlConnection = htmlURL.openConnection();//.filed                    HttpURLConnection conn = (HttpURLConnection) urlConnection;                    //获取网络资源的读取流                    InputStream is = conn.getInputStream();                    //判断网络资源响应是否成功                    if (conn.getResponseCode() == 200) {

//内存流 ByteArrayOutputStream                        ByteArrayOutputStream baos = new ByteArrayOutputStream();

byte[] buffer = new byte[20 * 1024];//每次读取最大内存大小20k，缓冲大小                        int len = -1; //每次读取的字节长度

//开始读取网路数据                        //检测文件下载的进度                        //1.获取网络资源的总长度                        int contentLength = conn.getContentLength();                        //2.声明当前已读取的资源长度，累加len                        int curLen = 0;

while ((len = is.read(buffer)) != -1) {                            //将读取的数据读取写入内存流中                            baos.write(buffer, 0, len);

//3.计算下载进度                            curLen += len;                            System.out.println(curLen + " " + contentLength);                            //4.计算下载进度                            int p = curLen  / contentLength;                            System.out.println("下载进度" + p + "%");                        }                        //下载完成,获取内存流中的数据                        byte[] bytes = baos.toByteArray();                        //将字节数组转成字符串，并打印到控制台//                        "hello".getBytes(); zifu->zijie                        String htmlContent = new String(bytes, "utf-8");                        writerFile(htmlContent, dirPath, fileName);//成员方法在下面                        System.out.println(htmlContent);                        //  System.out.println(htmlContent);                    }} catch (IOException e) {                    e.printStackTrace();                }            } catch (MalformedURLException e) {                e.printStackTrace();            }

}

//将文件存储在指定的path的文件中        private void writerFile(String htmlContent, String dirPath, String fileName) {            File dir = new File(dirPath);

FileWriter fileWriter = null;            try {                fileWriter = new FileWriter(new File(dir, fileName));            } catch (IOException e) {                e.printStackTrace();            }            try {                fileWriter.write(htmlContent);            } catch (IOException e) {                e.printStackTrace();            }            try {                fileWriter.close();            } catch (IOException e) {                e.printStackTrace();            }

}    }    public static void outputFile(String content,String dirPath,String fileName){        File dir=new File(dirPath);        try {            FileOutputStream fileOutputStream=new FileOutputStream(new File(dir,fileName));            fileOutputStream.write(content.getBytes("utf-8"));            fileOutputStream.close();        } catch (IOException e) {            e.printStackTrace();        }    }}