一个能自动、批量上传/下载文件的脚本

转自Simon的BLOG： http://blog.sina.com.cn/u/45d51ebc01000351

 

[引子]
工作中经常需要从不同的UNIX服务器上下载或上传文件，这些文件大都具有某些共性，比如说其目录结构相同，文件名相同等等，如果采取传统方式每次登录一台服务器下载或上传这些文件，随着服务器和文件数量的增长，所花费的时间将随之增之，且大都是重复性劳动，为此，我通过bsh编写了一个脚本，解决了上述问题。

[脚本描述]
脚本命名为auto-ftp-v3.sh，目前版本为v3.0，能够适合大部分文件上传/下载的需求，并在文件上传/下载过程中，对于大于5M的文件，能够显示其传输进度，但同时只允许该脚本的一个实例运行。

[用法]

基本语法：

# ./auto-ftp-v3.sh -s -f \
-d -m

说明：
-s server-list:为UNIX服务器列表文件，该文件中定义了UNIX服务器
名及用于登录的FTP用户和密码，一台服务器占一行，
其格式如下：

server-a username password

server-a:UNIX服务器主机名或IP地址
username:用于FTP登录的用户名，可以为空（使用缺省用户名）
password:FTP用户的密码，可以为空（使用缺省密码）

缺省的用户名和密码在脚本中通过如下变量指定，可根据实际更改：
DefaultUserName="explorer"
DefaultUserPwd="explorer"

-f file-list:为需要上传或下载的文件列表，在该文件中定义了上传
或下载的文件名，一个文件占一行，允许使用通配符
("*"和"?"),如果使用通配符，必须用双引号或单引号
括起来，如：
# more file-list
"*.gz" (匹配后缀名为.gz的所有文件）
'exp?' (匹配以exp开头的，且文件名长度为4的所有文件）
fileA
fileB （精确匹配单个文件，不需要单引号或双引号）

-d remote-dir:指定UNIX服务器上的远程目录，当为上传操作时，则是
指将文件上传到的远程目录，下载时则为文件存放在
服务器上的目录.

-m mget|mput:指定方向，文件上传时为mput,下载时为mget.



[例子]

# ./auto-ftp-v3.sh -s ./server-list -f ./file-list -d /tmp \
-m mget
# more file-list
"*.gz"
# more server-list
szxmr03-or
szxmr04-or osadmin 1rem3yq

脚本输出：

Initiator script and connect to target,Pls wait....

Get "*.gz" from szxmr03-or... [| 1%]
...

上述命令将自动从服务器szxmr03-or的szxmr04-or的/tmp目录下下载所有.gz文件到本地目录，使用缺省用户和密码登录到szxmr03-or.

[脚本]

#!/bin/sh
#==========================================================================
# Author:Simon Yuan/Sun Services/SZ Office/[e-mail]
# -s: server list file,include username & password,ONE Line for ONE Target
# -f: filename to be uploaded on remote server,ONE Line for ONE file
# -m: data tranfer direction
# version:3rd Edition
# Enchanced Function:
# *) Richer Color Sign
# *) Visual Data Transfer Process
# *) Data Tansfer To Multi- Target In Parallel
# *) Upload And Download Are Provided With A Script
#=========================================================================

trap 'AbnormalExit' 1 2 3 15

TERM=vt100
export TERM
LANG=C
export LANG

FtpOutFile="/tmp/ftp-out"
ErrSvrListFile="/tmp/server-list-err"
FtpFileSize="/tmp/ftp-filesize"
NullFile="`dirname $0`/null"

FileSize="0"

DefaultUserName="explorer"
DefaultUserPwd="explorer"

ScriptName=`basename $0`

if [ ` ps -ef | grep -w "$ScriptName" | grep -v grep | wc -l ` -gt 1 ];then
echo "\033[;31m\c"
echo "The script is running,multiple instances is denied!"
echo "\033[1;0m\c]"
exit 1
fi

if [ -f "$FtpOutFile".* ];then
rm "$FtpOutFile".*
fi

if [ -f "$ErrSvrListFile" ];then
rm $ErrSvrListFile
fi

if [ -f "$FtpFileSize" ];then
rm $FtpFileSize
fi

if [ ! -f "$NullFile" ];then
touch $NullFile
fi

touch $ErrSvrListFile

AbnormalExit ()
{
SetColor "DARKRED"
echo "\nThe Script received a break sign,Exit Abnormally!\n"
SetColor "RESET"
ClearTmpFile
exit
}

SetColor ()
{
case $1 in
"GREEN") echo "\033[1;32m\c]"
;;
"DARKGREEN") echo "\033[;32m\c"
;;
"RED") echo "\033[1;31m\c]"
;;
"DARKRED") echo "\033[;31m\c"
;;
"BLUE") echo "\033[1;34m\c]"
;;
"DARKBLUE") echo "\033[;34m\c]"
;;
"RESET") echo "\033[1;0m\c]"
;;
esac
}

Drawer ()
{
end_prompt="[DONE]"
_per=`expr $1 \* 100 / $2`
case `expr $_per / 2 % 4` in
0) _char="|" ;;
1) _char="/" ;;
2) _char="-" ;;
3) _char="\\" ;;
esac

SetColor "DARKBLUE"
printf "\r%s%40s%5s" "$3" [$_char $_per%]

if [ $1 -ge $2 ];then
SetColor "DARKGREEN"
printf "\r%s%40s%5s" "$3" $end_prompt
printf "\n"
SetColor "RESET"
fi
}

ClearTmpFile ()
{
rm -rf "$FtpOutFile".*
rm -rf $FtpFileSize
rm -rf $NullFile
}

GetLRFileSize ()
{

if [ "$METHOD" = "mget" ]; then
ftp -in << EOF > "$1" 2>/dev/null
open $str_svrname
user $USER_NAME $USER_PASSWD
cd $Target_DIR
dir $str_filename
bye
EOF
cat "$1" | /usr/bin/egrep "Not connected|Login failed" > /dev/null
if [ $? -ne 0 ]; then
grep -v "No such file" "$1" > /dev/null
#awk '{print $9}' "$1" | grep "$str_filename" > /dev/null

if [ $? -eq 0 ]; then
grep "Permission denied" "$1" > /dev/null
if [ $? -ne 0 ]; then
echo "$str_filename" |/usr/bin/egrep "\*|\?" > /dev/null
if [ $? -eq 0 ];then
cat "$1" | while read LINE
do
EachFileSize=`echo "$LINE" | awk '{print $5}'`
FileSize=`echo "$FileSize+$EachFileSize" | bc`
echo $FileSize > $FtpFileSize
done

FileSize=`cat $FtpFileSize`

else
FileSize=`awk '{print $5}' "$1"`
fi
fi
fi
fi

else #Put File to remote,put a null file to remote is to verify the specified user can access the remote directory

ftp -in << EOF > "$1" 2>/dev/null
open $str_svrname
user $USER_NAME $USER_PASSWD
cd $Target_DIR
mput $NullFile
delete $NullFile
bye
EOF
cat "$1" | /usr/bin/egrep "Not connected|Login failed" > /dev/null
if [ $? -ne 0 ]; then
echo "$str_filename" |/usr/bin/egrep "\*|\?" > /dev/null
if [ $? -eq 0 ]; then
str_filename=`echo "$str_filename" | sed -e 's/\"//g' | sed -e "s/\'//g"`

ls -l $str_filename > "$1" 2>/dev/null
if [ -s "$1" ];then
cat "$1" | while read LINE
do
EachFileSize=`echo "$LINE" | awk '{print $5}'`
FileSize=`echo "$FileSize+$EachFileSize" | bc`
echo $FileSize > $FtpFileSize
done

FileSize=`cat $FtpFileSize`
fi

else
if [ -s "$str_filename" ];then
FileSize=`ls -l $str_filename | awk '{print $5}'`
fi

fi
fi
fi

if [ $FileSize -lt 5242880 ]; then #file<5M,don't show guage
Guage="NotShow"
fi

# rm $1
}

AutoFtp ()
{
ftp -in << EOF > "$1" 2>/dev/null
open $str_svrname
user $USER_NAME $USER_PASSWD
cd $Target_DIR
hash
binary
$METHOD $str_filename
bye
EOF
}

Usage ()
{
echo "\n Usage:
\t`basename $0` -s -f
\t-d -m \n"
SetColor "DARKBLUE"
echo "Please make sure that the username and password is valid before run this script!"
SetColor "RESET"
if [ -f $NullFile ];then
rm $NullFile
fi
exit
}

if [ $# = "0" ];then
SetColor "DARKRED"
echo "Error,No any option specified!"
SetColor "RESET"
Usage $0
exit
fi

while getopts :s:f:u:p:d:m: args
do
case $args in
s) ops=$OPTARG
SERVER_LIST=`echo $ops`
if [ ! -s $SERVER_LIST ]
then
SetColor "DARKRED"
echo "Error,Server Lists File $SERVER_LIST don't exist or is an empty file!"
SetColor "RESET"
Usage $0
exit
fi
;;

f) ops=$OPTARG
FILE_LIST=`echo $ops`
if [ ! -s $FILE_LIST ]
then
SetColor "DARKRED"
echo "Error,File Lists File $FILE_LIST don't exist or is a zero file!"
SetColor "RESET"
Usage $0
exit
fi
;;

d) ops=$OPTARG
Target_DIR=`echo $ops`
;;
m) ops=$OPTARG
if [ $ops != "mget" ] && [ $ops != "mput" ];
then
SetColor "DARKRED"
echo "Error,Tranfer method must be 'mput' or 'mget' !"
SetColor "RESET"
Usage $0
exit
fi
METHOD=`echo $ops`
;;

?) Usage $0

esac
done

if [ $# -lt 8 ];then
SetColor "DARKRED"
echo "Error,Insufficient Options!"
SetColor "RESET"
Usage $0
fi

egrep "\*|\?" $FILE_LIST > /dev/null
if [ $? -eq 0 ]; then
egrep "\"|\'" $FILE_LIST > /dev/null
if [ $? -ne 0 ]; then
SetColor "DARKRED"
echo "When using wildcard (* or ?) to represent one or more files,please add \"\" for each line in filelist"
SetColor "RESET"
Usage $0
exit
fi
fi

COUNTER=1

echo "Initiator script and connect to target,Pls wait....\n"

#set -x
for str_filename in `cat $FILE_LIST`
do
cat $SERVER_LIST | while read item
do
echo "$item" | egrep "^#|^$" > /dev/null
if [ $? -eq 0 ];then
continue
fi

str_svrname=`echo $item | awk '{print $1}'`
USER_NAME=`echo $item | awk '{print $2}'`
USER_PASSWD=`echo $item | awk '{print $3}'`

if [ "Null$USER_NAME" = "Null" ];then
USER_NAME=$DefaultUserName
fi

if [ "Null$USER_PASSWD" = "Null" ];then
USER_PASSWD=$DefaultUserPwd
fi


if [ $METHOD = "mget" ];then
prompt="Get $str_filename from $str_svrname..."
else
prompt="Put $str_filename to $str_svrname..."
fi

FtpOut="$FtpOutFile.$COUNTER"
GetLRFileSize "$FtpOut"

cat "$FtpOut" | grep 'Not connected' > /dev/null
if [ $? -eq 0 ]; then
Err="[Failed,Can't connect the target host]."
SetColor "DARKRED"
echo "$prompt" "$Err"
SetColor "RESET"
grep "$str_svrname" "$ErrSvrListFile" > /dev/null
if [ $? -ne 0 ];then
echo "$str_svrname" >> $ErrSvrListFile
fi
Guage=""
continue
fi

cat "$FtpOut" | grep 'Connection timed out' > /dev/null
if [ $? -eq 0 ]; then
Err="[Failed,Connect to server timed out]."
SetColor "DARKRED"
echo "$prompt" "$Err"
SetColor "RESET"
grep "$str_svrname" "$ErrSvrListFile" > /dev/null
if [ $? -ne 0 ];then
echo "$str_svrname" >> $ErrSvrListFile
fi
Guage=""
continue
fi

cat "$FtpOut" | grep 'Login failed' > /dev/null
if [ $? -eq 0 ]; then
Err="[Failed,Username Or Password invalid,Or ftp request was rejected for $USER_NAME user]."
SetColor "DARKRED"
echo "$prompt" "$Err"
#printf "\n%s%20s" "$prompt" "$Err"
SetColor "RESET"
grep "$str_svrname" "$ErrSvrListFile" > /dev/null
if [ $? -ne 0 ];then
echo "$str_svrname" >> $ErrSvrListFile
fi
Guage=""
continue
fi

if [ "$FileSize" = "0" ]; then
Err="[Failed,File Can't be found or Permission denied when read targeted files]."
SetColor "DARKRED"
echo "$prompt" "$Err"
SetColor "RESET"
grep "$str_svrname" "$ErrSvrListFile" > /dev/null
if [ $? -ne 0 ];then
echo "$str_svrname" >> $ErrSvrListFile
fi
continue
fi

cat "$FtpOut" | grep 'Permission denied' > /dev/null
if [ $? -eq 0 ]; then
Err="[Failed,Permission denied when user $USER_NAME to access Remote DIR]."
SetColor "DARKRED"
echo "$prompt" "$Err"
SetColor "RESET"
grep "$str_svrname" "$ErrSvrListFile" > /dev/null
if [ $? -ne 0 ];then
echo "$str_svrname" >> $ErrSvrListFile
fi
Guage=""
continue
fi

AutoFtp "$FtpOut" &


if [ "Null$Guage" = "Null" ]; then
DataByte_Num=`echo "scale=2;$FileSize/8192/2"|bc`
if [ `echo $DataByte_Num | sed 's/^[0-9]*.//g'` = "00" ];then
DataByte_Num=`echo $DataByte_Num | sed 's/...$//g'`
else
DataByte_Num=`echo $DataByte_Num+1 |bc | sed 's/...$//g'`
fi

HashByte_Num=0

while true
do
Percent_Num=`echo "scale=2;$HashByte_Num/$DataByte_Num*100"|bc|sed 's/.00$//g'`
HashByte_Num=`grep "\#" $FtpOut | wc -c`
if [ "$Percent" = "100" ];then
Completed_Size=$FileSize
else
Completed_Size=`echo $HashByte_Num*8192*2|bc`
fi

#echo "[$Completed_Size/$FileSize]"

Drawer $Percent_Num 100 "$prompt"

if [ $Percent_Num -ge 100 ];then
break
fi

done
else
SetColor "DARKBLUE"
echo "$prompt \c"
SetColor "DARKGREEN"
echo " [DONE]"
SetColor "RESET"
Guage=""
fi

COUNTER=` expr $COUNTER + 1 `
FileSize="0"
done
done

if [ -s $ErrSvrListFile ];then
SetColor "DARKRED"
echo "\n\n--------------------"
echo "Error Hosts List were saved to file $ErrSvrListFile!"
SetColor "RESET"
fi

ClearTmpFile

echo "\nDONE.\n"