设置Hive Map和Reduce的数量增加分析速度

有些时候需要尝试优化Hive的Map和Reduce数量，来达到最佳运行状态，根据实际情况进行数量调整，可以更快速的得到结果。

1、增加map数量
首先调整上一步reducer生成文件数据，下面可以把reduce设置为160，即生成160个文件

set mapred.reduce.tasks=160;
create table test as
select * from temp
distribute by rand(123);

2、单纯调整map数量，增加map num

===================初步 filenum ：150 num , filesize: 1.2 G , map ：7 num, reduce : 100 num

====================================
hive (bigdata)> set mapreduce.job.reduces;
mapreduce.job.reduces=-1
hive (default)> set mapred.map.tasks;
mapred.map.tasks=200
hive (default)> set mapred.reduce.tasks;
mapred.reduce.tasks=-1 —(default： 2)
hive (default)> set dfs.block.size;
dfs.block.size=134217728
hive (bigdata)> set mapred.min.split.size;
mapred.min.split.size=1
hive (default)> set mapred.max.split.size;
mapred.max.split.size=256000000
drop table default.tb_user_terminal_test;
create table default.tb_user_terminal_test as select sum(mdn),usp,times,start_time from bigdata.tb_user_terminal_udp_s2 group by mdn,times,start_time,usp;
— Time taken: 74.709 seconds

====================

hive (bigdata)> set mapred.map.tasks;
mapred.map.tasks=160
hive (bigdata)> set mapreduce.job.reduces;
mapreduce.job.reduces=100
hive (bigdata)> set mapred.reduce.tasks;
mapred.reduce.tasks=150
hive (bigdata)> set dfs.block.size;
dfs.block.size=16777216
hive (bigdata)> set mapred.min.split.size;
mapred.min.split.size=1
hive (bigdata)> set mapred.max.split.size;
mapred.max.split.size=2560000
drop table default.tb_user_terminal_test;
create table default.tb_user_terminal_test as select sum(mdn),usp,times,start_time from bigdata.tb_user_terminal_udp_s2 group by mdn,times,start_time,usp;
— Time taken: 126.13 seconds

===================

hive (default)> set mapreduce.job.reduces;
mapreduce.job.reduces=100
hive (default)> set mapred.map.tasks;
mapred.map.tasks=200
hive (default)> set mapred.reduce.tasks;
mapred.reduce.tasks=100
hive (default)> set dfs.block.size;
dfs.block.size=134217728
hive (default)> set mapred.min.split.size;
mapred.min.split.size=1
hive (default)> set mapred.max.split.size;
mapred.max.split.size=25600000
drop table default.tb_user_terminal_test;
create table default.tb_user_terminal_test as select sum(mdn),usp,times,start_time from bigdata.tb_user_terminal_udp_s2 group by mdn,times,start_time,usp;
— Time taken: 47.179 seconds

===================

hive (default)> set mapreduce.job.reduces;
mapreduce.job.reduces=100
hive (default)> set mapred.map.tasks; —
mapred.map.tasks=200
hive (default)> set mapred.reduce.tasks; —
mapred.reduce.tasks=58
hive (default)> set dfs.block.size;
dfs.block.size=134217728 —
hive (default)> set mapred.min.split.size;
mapred.min.split.size=1
hive (default)> set mapred.max.split.size;
mapred.max.split.size=25600000 —
drop table default.tb_user_terminal_test;
create table default.tb_user_terminal_test as select sum(mdn),usp,times,start_time from bigdata.tb_user_terminal_udp_s2 group by mdn,times,start_time,usp;
— Time taken: 40.749 seconds

======================最终调整=== filesize : 1.2g, map ：150 num, reduce : 58 num , file: 150 num

========================
hive (default)> set mapreduce.job.reduces;
mapreduce.job.reduces=100
hive (default)> set mapred.map.tasks;
mapred.map.tasks=200
hive (default)> set mapred.reduce.tasks;
mapred.reduce.tasks=58
hive (default)> set hive.merge.mapredfiles;
hive.merge.mapredfiles=false
hive (default)> set dfs.block.size;
dfs.block.size=134217728
hive (default)> set mapred.min.split.size;
mapred.min.split.size=1
hive (default)> set mapred.max.split.size;
mapred.max.split.size=4560000
hive (default)> set hive.groupby.skewindata;
set hive.groupby.skewindata=true
drop table default.tb_user_terminal_test;
create table default.tb_user_terminal_test as select sum(mdn),usp,times,start_time from bigdata.tb_user_terminal_udp_s2 group by mdn,times,start_time,usp;
—Time taken: 42.903 seconds

由于我们需求是没有reducer，为了提高集群资源利用率，手动提高了map的数量！
结论：提高了map ：7-->150 num，最后平均跑2h的任务，缩减平均10min!