获取hadoop3.0.0MapReduce的容器日志

#!/bin/bash
resourceManager=cm1
jobId=$1
firstLineNumber=128
afterLineNumber=6
rename=true
`curl http://${resourceManager}:19888/jobhistory/attempts/${jobId}/m/SUCCESSFUL > a.xml`
urls=`cat a.xml | grep container | awk -F"'" '{print $10}'`
if [ ! -d ./${jobId}_logs ];then
    mkdir ${jobId}_logs
fi
for url in ${urls[@]};do
    #echo $url
    mapName=`echo ${url} | awk -F'/' '{print $5}'`
    #echo $mapName
    echo "curl http://${resourceManager}:19888${url}/syslog/?start=0 > ${jobId}_logs/${mapName}.log"
    (curl http://${resourceManager}:19888${url}/syslog/?start=0 > ${jobId}_logs/${mapName}.log)
    (sed -i "1,${firstLineNumber}d" ${jobId}_logs/${mapName}.log)
    lines=`cat ${jobId}_logs/${mapName}.log | wc -l`
    tmp=`expr $lines - $afterLineNumber`
    afterDelNum=`expr $tmp - 1`
    (sed -i "${afterDelNum},${lines}d" ${jobId}_logs/${mapName}.log)
done
if [ "$rename" = true ];then
    newFileName=$2
    (mv ${jobId}_logs ${newFileName}_logs)
    (zip -r ${newFileName}_logs.zip ${newFileName}_logs)
else
  (zip -r ${jobId}_logs.zip ${jobId}_logs)
fi
(rm -rf a.xml)

#!/bin/bash
ids=$(cat jobIDs.txt)
id_list=(${ids})
names=$(cat jobNames.txt)
name_list=(${names})
len=`expr ${#id_list[@]} - 1`
for i in `seq 0 $len`;do
    echo "sh getJobHistoryLogs.sh ${id_list[$i]} ${name_list[$i]}"
    (sh getJobHistoryLogs.sh ${id_list[$i]} ${name_list[$i]})
done

job_1642669840858_58970 job_1642669840858_58984 job_1642669840858_58979 job_1642669840858_58985 job_1642669840858_58964 job_1642669840858_58793 job_1642669840858_58977 job_1642669840858_58965 job_1642669840858_58795 job_1642669840858_58976

snapshotScanMR-NSSFCJSFW_BAK snapshotScanMR-XSDQ_BAK snapshotScanMR-QSFCJSFW_BAK snapshotScanMR-XSFCJSFW_BAK snapshotScanMR-MHSDQ_BAK snapshotScanMR-NSSFJ_BAK snapshotScanMR-PLSJ3_BAK snapshotScanMR-MHSFCJSFW_BAK snapshotScanMR-MHSFJ_BAK snapshotScanMR-PLSJ2_BAK

开启yarn日志聚合功能的日志收集

#!/bin/bash
outPath=logs
if [ ! -d logs ];then
    (mkdir logs)
fi
ids=$(cat jobIds.txt)
id_list=(${ids})
names=$(cat jobNames.txt)
name_list=(${names})
len=`expr ${#id_list[@]} - 1`
for i in `seq 0 $len`;do
    jobId1=`echo ${id_list[$i]} | awk -F'_' '{print $2}'`
    jobId2=`echo ${id_list[$i]} | awk -F'_' '{print $3}'`
    echo "yarn logs -applicationId application_${jobId1}_${jobId2} > logs/${name_list[$i]}.log" 
    (yarn logs -applicationId application_${jobId1}_${jobId2} > logs/${name_list[$i]}.log) 
done