#!/bin/bash

# ~/datax/bin/start_datax.sh

. ~/.bashrc

echo $#

if [ $# -lt 1 ]

then

    echo "need job config file"

    exit

fi

jobname="$1"

taskname=`echo $(basename $jobname .xml)`

echo "$jobname,taskname=$taskname"

DATAX_HOME=/home/work/datax

timenow=` date +'%H%M%S'`

cd $DATAX_HOME

outputfile="${DATAX_HOME}/log/${taskname}.log"

errorfile="${DATAX_HOME}/log/${taskname}.error"

rm ${outputfile} -f

python ${DATAX_HOME}/bin/datax.py ${jobname} >> ${outputfile} 2>${errorfile}

if [ $? -eq 0 ]

then

    tail  -n32 ${outputfile}

    exit 0

else

    tail  -n32 ${outputfile}

    exit 102

fi

tail  -n32 ${outputfile}

exit 103

**************************

#!/bin/bash

# core_incr.sh

. ~/.bashrc

start_date=`date -d "1 days ago" +%Y-%m-%d`

if [ $# -gt 0  ]

then

    start_date=$1

fi

end_date=`date -d "${start_date} 1 days " "+%Y-%m-%d"`

schema="core"

tbname="t_trans"

echo ${start_date} ${end_date}

example="/home/work/datax/job/core/example"

tempdir="/home/work/datax/job/core"

basepath=$(cd `dirname $0`; pwd)

cd $basepath

ARR=(1 2 3 4 5 6 7 8)

for ID in ${ARR[*]}

do

    jsonfile="${tempdir}/${schema}_${tbname}_$ID"

    oneexample="${example}/t_trans_example_$ID"

    sed "s/START_DATE/${start_date}/g;s/END_DATE/${end_date}/g" ${oneexample} > ${jsonfile}

done

echo "${jsonfile} done"

ARR=(1 2 3 4 5 6 7 8)

for ID in ${ARR[*]}

do

    echo "~/datax/log/${schema}_${tbname}_$ID.log"

    ~/datax/bin/start_datax.sh /home/work/datax/job/core/${schema}_${tbname}_$ID > /home/work/daiyuanpei/core_incr_$ID.out && bash /home/work/datax/job/dataxload_core.sh core t_trans p$ID >> /home/work/daiyuanpei/core_incr_$ID.out 2>&1 &

    sleep 3m

done

echo "core_incr.sh done"

**************************

#!/bin/bash

# /home/work/datax/job/dataxload_core.sh

. ~/.bashrc

if [ $# -lt 3 ]

then

    echo "./loaddata schema tablename hostindex"

    exit

fi

export LANG="zh_CN.UTF-8"

export LC_ALL="zh_CN.UTF-8"

schema=$1

tablename=$2

hostindex=$3

data_path=" /home/work/datax/ext/core/${tablename}_${hostindex}"

basepath=$(cd `dirname $0`; pwd)

cd ${basepath}

if [ ! -e ${data_path} ];then

 echo "no exist filepath:${data_path}"

 exit

fi

LOAD_SQL="insert into ${schema}.${tablename} select to_date(F_create_time::text,'YYYY-MM-DD'),* from pay_data_ext.r_ext_${tablename}_${hostindex};"

tempfile="${basepath}/temp.log"

export PGPASSWORD=************

/usr/bin/psql -h **.***.**.*** -p 5432 -d qq_gp_db -U gp_rd -c "$LOAD_SQL" > ${tempfile} 2>&1

if [ $? -eq 0 ]

   then

        logmsg=`cat ${tempfile}`

        echo "load to success ${tablename}:${logmsg}"

        emailTitle="[云上][${schema}.${tablename}_${hostindex}] load success"

        emailContent=`echo "datax load to gppay success: ${schema}.${tablename}_${hostindex}: ${logmsg}"`

     else

        logmsg=`cat ${tempfile}`

        msg=`echo "datax load to gppay failed:${tablename}:${logmsg}"`

        emailTitle="[云上][${schema}.${tablename}_${hostindex}] load failed"

        emailContent=`echo "datax load to gppay failed: ${schema}.${tablename}_${hostindex}: ${logmsg}"`

sleep 9s

nohup /bin/bash /home/work/daiyuanpei/retry_dataxload_core.sh {$schema} ${tablename} ${hostindex} > /home/work/daiyuanpei/retry_${tablename}_${hostindex}.out 2>&1 &

fi

echo ${emailContent} | mail -s "${emailTitle}" daiyp@qq.com -c "daiyuanpei@qq.com daiyuankai@qq.com"

**************************

#!/bin/bash

# monitor.sh

# 3 8 * * * (cd /home/work/daiyuanpei; bash ./monitor.sh > ct_monitor.out 2>&1 &)

. ~/.bashrc

current_date=`date +%Y-%m-%d`

echo ${current_date}

basepath=$(cd `dirname $0`; pwd)

cd $basepath

BAK=/home/work/daiyuanpei/monitor

if [ "`ls -A ${BAK}`" = "" ];

then

echo "${BAK} is empty"

else

        echo "${BAK} is not empty"

/usr/local/bin/scp -i /home/work/.ssh/id_rsa /home/work/daiyuanpei/monitor/t_* work@**.***.**.***:/home/work/daiyuanpei/monitor

echo "file move done"

rm ./monitor/t_*

echo "empty folder done"

fi