执行大数据分析脚本

#!/bin/bash
var1=$1       （备注：传递位置变量20160726）
var2=$2    （备注：传递位置变量20160730）
date1=${var1:0:4}"-"${var1:4:2}"-"${var1:6:2}" 00:00:00"   （备注：调用变量，转变为字符串）
date2=${var2:0:4}"-"${var2:4:2}"-"${var2:6:2}" 00:00:00"
time1=`date -d "$date1" +%s`    （备注：把字符串转换为时间戳）
time2=`date -d "$date2" +%s`
while [ $time1 -le $time2 ]      （备注：对时间戳进行比较）
do
date=`date -d @$time1 "+%Y%m%d"`  （备注：调用变量，把时间戳变为字符串）
Date=`date -d @$time1 "+%Y-%m-%d"`
time3=`expr $time1 - 86400`   （备注：减去一天时间）
DATE_1=`date -d @$time3 "+%Y%m%d"`

Path=/home/hadoop/hoch/data/beijing
if [ -f $Path/gehua/$Date ];then
 cp $Path/gehua/$date.tar.bz2 $Path/gh
tar -jxv -C $Path/gh -f $Path/gh/$date.tar.bz2 $date/ipanel/ars*
cd $Path/gh/$date
ls $Path/gh/$date/ipanel/ars*/$Date.zip | xargs -n1 unzip 
cat $Path/gh/$date/$Date/*>>$Path/gh/${date}ars.txt
hadoop fs -mkdir /source/110000/$date
hadoop fs -put $Path/gh/${date}ars.txt /source/110000/$date


##shuffle
spark-submit --class com.starv.spark.format.live.BJGeHuaFormat   ~/data2/jiangmeng/spark-1.0-SNAPSHOT.jar  $Date#$Date  /source/110000/   /business/live/$date


spark-submit --class com.starv.spark.live.asyn.ArriveNum2H1   ~/data2/jiangmeng/spark-1.0-SNAPSHOT.jar  /business/live/$date/  110000 


spark-submit --class com.starv.spark.live.asyn.ArriveNumNewmedia2H   ~/data2/jiangmeng/spark-1.0-SNAPSHOT.jar  /business/live/$date/  110001


spark-submit --class com.starv.spark.live.asyn.ViewTime24H  ~/data2/jiangmeng/spark-1.0-SNAPSHOT.jar  /business/live/$date/   110000


spark-submit --class com.starv.spark.live.asyn.UserNum  ~/data2/jiangmeng/spark-1.0-SNAPSHOT.jar   hdfs:///business/live/ 110000\|110001  $Date 90


spark-submit --class com.starv.spark.live.asyn.IdStatisticNew  ~/data2/jiangmeng/spark-1.0-SNAPSHOT.jar  hdfs:///business/live/$date/ 110000\|110001 /business/view/$DATE_1/110000/IdBefore /business/view/$date/110000/ $Date


else
exit
fi


rm -rf $Path/gh/${date}ars.txt
rm -rf $Path/gehua/$Date
rm -rf $Path/gh/$date
time1=`expr $time1 + 86400`            （备注：每循环一次增加一天）
done