

datax全流程脚本
2018-01-22 20:25:07
- 来源:
- 作者:代元培
- 类别:原创
- 浏览:9139 次
#!/bin/bash
# ~/datax/bin/start_datax.sh
. ~/.bashrc
echo $#
if [ $# -lt 1 ]
then
echo "need job config file"
exit
fi
jobname="$1"
taskname=`echo $(basename $jobname .xml)`
echo "$jobname,taskname=$taskname"
DATAX_HOME=/home/work/datax
timenow=` date +'%H%M%S'`
cd $DATAX_HOME
outputfile="${DATAX_HOME}/log/${taskname}.log"
errorfile="${DATAX_HOME}/log/${taskname}.error"
rm ${outputfile} -f
python ${DATAX_HOME}/bin/datax.py ${jobname} >> ${outputfile} 2>${errorfile}
if [ $? -eq 0 ]
then
tail -n32 ${outputfile}
exit 0
else
tail -n32 ${outputfile}
exit 102
fi
tail -n32 ${outputfile}
exit 103
**************************
#!/bin/bash
# core_incr.sh
. ~/.bashrc
start_date=`date -d "1 days ago" +%Y-%m-%d`
if [ $# -gt 0 ]
then
start_date=$1
fi
end_date=`date -d "${start_date} 1 days " "+%Y-%m-%d"`
schema="core"
tbname="t_trans"
echo ${start_date} ${end_date}
example="/home/work/datax/job/core/example"
tempdir="/home/work/datax/job/core"
basepath=$(cd `dirname $0`; pwd)
cd $basepath
ARR=(1 2 3 4 5 6 7 8)
for ID in ${ARR[*]}
do
jsonfile="${tempdir}/${schema}_${tbname}_$ID"
oneexample="${example}/t_trans_example_$ID"
sed "s/START_DATE/${start_date}/g;s/END_DATE/${end_date}/g" ${oneexample} > ${jsonfile}
done
echo "${jsonfile} done"
ARR=(1 2 3 4 5 6 7 8)
for ID in ${ARR[*]}
do
echo "~/datax/log/${schema}_${tbname}_$ID.log"
~/datax/bin/start_datax.sh /home/work/datax/job/core/${schema}_${tbname}_$ID > /home/work/daiyuanpei/core_incr_$ID.out && bash /home/work/datax/job/dataxload_core.sh core t_trans p$ID >> /home/work/daiyuanpei/core_incr_$ID.out 2>&1 &
sleep 3m
done
echo "core_incr.sh done"
**************************
#!/bin/bash
# /home/work/datax/job/dataxload_core.sh
. ~/.bashrc
if [ $# -lt 3 ]
then
echo "./loaddata schema tablename hostindex"
exit
fi
export LANG="zh_CN.UTF-8"
export LC_ALL="zh_CN.UTF-8"
schema=$1
tablename=$2
hostindex=$3
data_path=" /home/work/datax/ext/core/${tablename}_${hostindex}"
basepath=$(cd `dirname $0`; pwd)
cd ${basepath}
if [ ! -e ${data_path} ];then
echo "no exist filepath:${data_path}"
exit
fi
LOAD_SQL="insert into ${schema}.${tablename} select to_date(F_create_time::text,'YYYY-MM-DD'),* from pay_data_ext.r_ext_${tablename}_${hostindex};"
tempfile="${basepath}/temp.log"
export PGPASSWORD=************
/usr/bin/psql -h **.***.**.*** -p 5432 -d qq_gp_db -U gp_rd -c "$LOAD_SQL" > ${tempfile} 2>&1
if [ $? -eq 0 ]
then
logmsg=`cat ${tempfile}`
echo "load to success ${tablename}:${logmsg}"
emailTitle="[云上][${schema}.${tablename}_${hostindex}] load success"
emailContent=`echo "datax load to gppay success: ${schema}.${tablename}_${hostindex}: ${logmsg}"`
else
logmsg=`cat ${tempfile}`
msg=`echo "datax load to gppay failed:${tablename}:${logmsg}"`
emailTitle="[云上][${schema}.${tablename}_${hostindex}] load failed"
emailContent=`echo "datax load to gppay failed: ${schema}.${tablename}_${hostindex}: ${logmsg}"`
sleep 9s
nohup /bin/bash /home/work/daiyuanpei/retry_dataxload_core.sh {$schema} ${tablename} ${hostindex} > /home/work/daiyuanpei/retry_${tablename}_${hostindex}.out 2>&1 &
fi
echo ${emailContent} | mail -s "${emailTitle}" daiyp@qq.com -c "daiyuanpei@qq.com daiyuankai@qq.com"
**************************
#!/bin/bash
# monitor.sh
# 3 8 * * * (cd /home/work/daiyuanpei; bash ./monitor.sh > ct_monitor.out 2>&1 &)
. ~/.bashrc
current_date=`date +%Y-%m-%d`
echo ${current_date}
basepath=$(cd `dirname $0`; pwd)
cd $basepath
BAK=/home/work/daiyuanpei/monitor
if [ "`ls -A ${BAK}`" = "" ];
then
echo "${BAK} is empty"
else
echo "${BAK} is not empty"
/usr/local/bin/scp -i /home/work/.ssh/id_rsa /home/work/daiyuanpei/monitor/t_* work@**.***.**.***:/home/work/daiyuanpei/monitor
echo "file move done"
rm ./monitor/t_*
echo "empty folder done"
fi
- 1
- 2
- 3
- 关键词标签: bash
- 责任编辑:代元培

