Chinaunix首页 | 论坛 | 博客
  • 博客访问: 376293
  • 博文数量: 1051
  • 博客积分: 53280
  • 博客等级: 大将
  • 技术积分: 6670
  • 用 户 组: 普通用户
  • 注册时间: 2008-09-09 13:21
文章分类

全部博文(1051)

文章存档

2011年(1)

2008年(1050)

我的朋友

分类:

2008-09-09 16:45:50

    同时多个文件,不过单文件没有多线程的并且没有断点续传功能,继续完善:

    view plaincopy to clipboardprint?
    package com.util.file;

    public class Files {

        /***
         * 获取应用程序的根目录
         * @return 应用程序根目录
         */
        public static String getSysPath(){
        return  System.getProperty("user.dir");
        }


    }

    package com.util.file;

    public class Files {

        /***
         * 获取应用程序的根目录
         * @return 应用程序根目录
         */
        public static String getSysPath(){
     return  System.getProperty("user.dir");
        }


    }view plaincopy to clipboardprint?


     view plaincopy to clipboardprint?
   

package com.core.crawl;

    import java.io.File;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.net.HttpURLConnection;
    import java.net.URL;

    import com.core.http.Http;

    public class WebSpider implements Runnable{

        private Http http = new Http();

        private String webAddress = "";
        private String destFile = "";

        public void setWebAddress(String webAddress){
        this.webAddress = webAddress;
        }

        public void setDestFile (String destFile){
        this.destFile = destFile;
        }

        public boolean download() throws IOException, InterruptedException {

        HttpURLConnection httpConn = null;

        try {
            URL url = new URL(webAddress);

            httpConn = (HttpURLConnection) url.openConnection();
            httpConn.setRequestMethod("GET");
            httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (; U; NT 5.1; zh-CN; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14");
            InputStream in = httpConn.getInputStream();
            String fileType = http.fileType(httpConn.getContentType());
            System.out.println(fileType);
            FileOutputStream out = new FileOutputStream(new File(destFile + fileType));
            int chByte = in.read();
            while (chByte != -1) {
            out.write(chByte);
            //System.out.println(chByte);
            chByte = in.read();
            }
        } catch (Exception ex) {
            System.out.println(ex.toString());
        } finally {
            httpConn.disconnect();
        }
        return true;
        }

        public void run() {
        try {
            //System.out.println(Thread.currentThread().getName());
            download();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        }
    }
   

    view plaincopy to clipboardprint?package com.core.crawl;     import java.io.File;   import java.io.FileOutputStream;   import java.io.IOException;   import java.io.InputStream;   import java.net.HttpURLConnection;   import java.net.URL;     import com.core.http.Http;     public class WebSpider implements Runnable{              private Http http = new Http();         private String webAddress = "";       private String destFile = "";              public void setWebAddress(String webAddress){       this.webAddress = webAddress;       }              public void setDestFile (String destFile){       this.destFile = destFile;       }              public boolean download() throws IOException, InterruptedException {         HttpURLConnection httpConn = null;         try {           URL url = new URL(webAddress);                    httpConn = (HttpURLConnection) url.openConnection();           httpConn.setRequestMethod("GET");           httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14");           InputStream in = httpConn.getInputStream();           String fileType = http.fileType(httpConn.getContentType());           System.out.println(fileType);           FileOutputStream out = new FileOutputStream(new File(destFile + fileType));           int chByte = in.read();           while (chByte != -1) {           out.write(chByte);           //System.out.println(chByte);           chByte = in.read();           }       } catch (Exception ex) {           System.out.println(ex.toString());       } finally {           httpConn.disconnect();       }       return true;       }         public void run() {       try {           //System.out.println(Thread.currentThread().getName());           download();           } catch (IOException e) {           e.printStackTrace();       } catch (InterruptedException e) {           e.printStackTrace();       }       }   }  package com.core.crawl;

    import java.io.File;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.net.HttpURLConnection;
    import java.net.URL;

    import com.core.http.Http;

    public class WebSpider implements Runnable{

        private Http http = new Http();

        private String webAddress = "";
        private String destFile = "";

        public void setWebAddress(String webAddress){
     this.webAddress = webAddress;
        }

        public void setDestFile (String destFile){
     this.destFile = destFile;
        }

        public boolean download() throws IOException, InterruptedException {

     HttpURLConnection httpConn = null;

     try {
         URL url = new URL(webAddress);

         httpConn = (HttpURLConnection) url.openConnection();
         httpConn.setRequestMethod("GET");
         httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14");
         InputStream in = httpConn.getInputStream();
         String fileType = http.fileType(httpConn.getContentType());
         System.out.println(fileType);
         FileOutputStream out = new FileOutputStream(new File(destFile + fileType));
         int chByte = in.read();
         while (chByte != -1) {
      out.write(chByte);
      //System.out.println(chByte);
      chByte = in.read();
         }
     } catch (Exception ex) {
         System.out.println(ex.toString());
     } finally {
         httpConn.disconnect();
     }
     return true;
        }

        public void run() {
     try {
         //System.out.println(Thread.currentThread().getName());
         download();
     } catch (IOException e) {
         e.printStackTrace();
     } catch (InterruptedException e) {
         e.printStackTrace();
     }
        }
    }

    view plaincopy to clipboardprint?
   

package com.core.crawl;

    import java.io.IOException;

    import com.util.file.Files;

    public class Crawl {

        /**
         * @param args
         * @throws IOException
         * @throws InterruptedException
         */
        public static void main(String[] args) throws IOException, InterruptedException {

        long begin = System.currentTimeMillis();
        WebSpider spider2 = new WebSpider();
        WebSpider spider1 = new WebSpider();
        spider1.setWebAddress("");
        spider1.setDestFile(Files.getSysPath() + "/"+"spider1.");

        spider2.setWebAddress("");
        spider2.setDestFile(Files.getSysPath() + "/"+"spider2.");

        Thread t1 = new Thread(spider1);
        Thread t2 = new Thread(spider2);
        t1.start();
        t2.start();

        t1.join();
        t2.join();

        System.out.println("the end");
        System.out.println(System.currentTimeMillis() - begin);
        }

 

    }


   
 

   
测试通过:

   

【责编:landy】

--------------------next---------------------

阅读(124) | 评论(0) | 转发(0) |
给主人留下些什么吧!~~