wget 多进程下载

#!/bin/bash
PATH1=/mnt/tempdisk3
cd $PATH1

count=1
for file in $(awk '{print $1}' /mnt/tempdisk3/to_wenming_url_list)
  do
      
      url=`awk '/'"$file"'/{print $2}' /mnt/tempdisk3/to_wenming_url_list`
       F=$[($count)%19]
       mkdir -p   $PATH1/data/$F
       wget -c  -O $PATH1/data/$F/$file.mp3  $url  &
       pscount=$(ps -ef |grep -v "grep" | grep -c wget)
       until [ $pscount -le 5 ]
                do
                sleep 2
                   pscount=$(ps -ef |grep -v "grep" | grep -c wget)
                done
        (( count=count+1 ))
done

 

 

#------------------------------

 

由于源文件mp3量太大,都放一个目录效率不高,用%取膜方式建19个目录 然后存到各个目录里

wget 进程载控制在5个(可以自己调).

#!/bin/bash
PATH1=/mnt/tempdisk3
cd $PATH1

count=1
for file in $(awk '{print $1}' /mnt/tempdisk3/to_wenming_url.list2_new)
  do
      
      url=`awk '/'"$file"'/{print $2}' /mnt/tempdisk3/to_wenming_url.list2_new`
       F=$[($count)%19]
       mkdir -p   $PATH1/data2/$F
       wget -c  -O $PATH1/data2/$F/$file.mp3  $url  &
       sed -i '/'"$file"'/d' /mnt/tempdisk3/to_wenming_url.list2_new
       pscount=$(ps -ef |grep -v "grep" | grep -c wget)
       until [ $pscount -le 15 ]
                do
                sleep 2
                   pscount=$(ps -ef |grep -v "grep" | grep -c wget)
                done
        (( count=count+1 ))
done
   

 

 

以下是改进版：

 

#!/bin/bash
PATH1=$1
PATH2=$2
name=$3
cd $PATH1
count=1
while read line
  do
      file=$(echo $line |awk  '{print $1}')
      url=$(echo $line |awk  '{print $2}')
      extend=$(echo $line |awk -F. '{print $NF}')
       F=$[($count)%19]
       mkdir -p   $PATH2/data/$F
       wget -c -t1 -O $PATH2/data/$F/$file.$extend  $url  &
       if [ $? = 0 ]; then
          sed -i '/^'"$file"'/d' $PATH1/$name
       fi
       pscount=$(ps -ef |grep -v "grep" | grep -c wget)
       until [ $pscount -le 20 ]
                do
                sleep 2
                   pscount=$(ps -ef |grep -v "grep" | grep -c wget)
                done
        (( count=count+1 ))
done < out_url_song.csv