微型爬虫代码

io.open()

function spider(domain,count = 0){
    import inet.http;
    import web.url
    
    var domain_s = web.url.split(domain)
    
    io.print( "开始任务",domain );
/*
    if( io.exist( io.fullpath("/website/" + domain_s.host + ".txt") ) ){
        io.print( "已完成的任务，跳过 ",domain )
        return 0;
    }
*/
    
    //声明HTTP对象
    var http = inet.http()
    
    if( ..string.startWith(domain,"http://") == false ){
        domain = "http://" + domain;
    }
    
    //url队列
    var urlqueue = {domain};
    var urldic = { }
    var pages = 0;
    
    //添加到URL队列
    var function addurl(url){
      if( url == domain)
          return;
          
      if(!url)
          return;
          
      //排重处理
      if( not urldic[url] ){
          urldic[url] = true;
          table.push( urldic,url);
          if( #urldic > 1000){ //缓冲区限制

              urldic[ table.remove(urldic) ] = null;
          }
          
          table.push( urlqueue,url);
      }
      else{
          //io.print("已排重处理",url)
      }
      
    }
    
    var tpagecontent = {}

    while(true){
      sleep(10)
      pages ++;
      if( count && ( pages > count) )
          break;
          
      var url = table.remove( urlqueue )
      if(url=="")continue ;
      if(!url ) break;
          
      io.print( url,'\n' )
      var str,errmsg = http.down(url);
      if(!str)
          continue ;
          
      ..table.push( tpagecontent, str );
      
      for href in string.gmatch( str,"\s*href\s*=\s*\""?\