针对脚本的注解和整体构思，我会放到脚本之后为大家详解。
#!/bin/bash##Thisscriptisusedtograbthedataonthespecifiedindustrywebsites#Writtenbysunsky#Mail:274546888@qq.com#Date:2014-09-143:06:00#if[`echo$UID`!=0];thenecho’Pleaseusetheroottoexecutethescript!’fiif[!-f/dataimg/years];thenecho’Pleasegivedatefile,thefilepathfor/dataimg/years.’fiif[!-d$TMP_DIR];thenmkdir-p$TMP_DIRfiif[!-d$URL_MD5_DIR];thenmkdir-p$URL_MD5_DIRfiif[!-d$HTML_DIR];thenmkdir-p$HTML_DIRfiROOT_DIR=”/dataimg”#指定脚本运行根目录TMP_DIR=”$ROOT_DIR/tmp”#生成商品详细页url之前的临时数据存放目录URL_MD5_DIR=”$ROOT_DIR/url_md5″#记录商品详细页url的MD5值的目录HTML_DIR=”$ROOT_DIR/html”#存放下载下来的商品详细页目录URL_MD5=”$URL_MD5_DIR/md5.$year”#负责记录商品详细页url的md5值WEB_URL=”https://www.redhat.sx/”#所爬网站的主页urlREPORT=”$ROOT_DIR/report”#负责记录采集的url综合信息CURL=”curl-A’Mozilla/5.0(WindowsNT6.1;WOW64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/37.0.2062.102Safari/537.36′–referer”OPT0=”/dataimg/years”#年份信息OPT1=”$TMP_DIR/${X1_MD5}”#品牌信息OPT2=”$TMP_DIR/${X1_MD5}_${X2_MD5}”#车型信息OPT3=”$TMP_DIR/${X1_MD5}_${X2_MD5}_${X3_MD5}”#装饰信息OPT4=”$TMP_DIR/${X1_MD5}_${X2_MD5}_${X3_MD5}_${X4_MD5}”#部位分类信息OPT5=”$TMP_DIR/${X1_MD5}_${X2_MD5}_${X3_MD5}_${X4_MD5}_${URL_LIST_MD5}”#商品详情页url信息FIFO_FILE=”/tmp/$$.fifo”mkfifo$FIFO_FILEexec9<>$FIFO_FILErm-f$FIFO_FILEnum=10for((i=0;i<$num;i++));doechodone>&9whilereadX1;do{URL1=”${WEB_URL}/model/YMMTSelects.cfc?method=getMakes&PassYear=$X1″X1_MD5=`echo$URL1|cksum|cut-d”-f1`if!ls$OPT1>&/dev/null;then$CURL-s$URL1|awk’BEGIN{RS=”<“}{print$0}’|awk-F’>”{print$2}’|sed’1,9d’|sed’$d’|grep-v’^$’>$OPT1fiwhilereadX2;doX2=`echo$X2|sed’s##%20#g’`URL2=”${URL1}&PassMakeName=$X2″#X2_MD5=`echo$URL|cksum|cut-d”-f1`if!ls$OPT2>&/dev/null;then$CURL-s$URL2|awk’BEGIN{RS=”<“}{print$0}’|awk-F’>”{print$2}’|sed’1,6d’|sed’$d’|grep-v’^$’>$OPT2fiwhilereadX3;doX3=`echo$X3|sed’s#[[:space:]]#%20#g’`URL3=”${URL2}&PassModel=$X3″X3_MD5=`echo$URL3|cksum|cut-d”-f1`if!ls$OPT3>&/dev/null;then$CURL-s$URL3|sed’s#[[:space:]]##g’|awk’BEGIN{RS=”<|=|>”}{print$0}’|egrep’^[0-9]+$’>$OPT3fiwhilereadX4;doX4=`echo$X4|sed’s##%20#g’`URL4=”${URL3}&PassVehicleID=$X4″X4_MD5=`echo$URL4|cksum|cut-d”-f1`if!ls”${OPT4}”>&/dev/null;then$CURL-s$URL4|awk’BEGIN{RS=”<“}{print$0}’|awk-F'[>;]”{print$2}’|sed-e’1,3d’-e’$d’-e’/^$/d’>$OPT4fiwhilereadX5;doX5=`echo$X5|sed’s##%20#g’`URL_LIST=”${WEB_URL}index.cfm?fuseaction=store.sectionSearch&YMMTyears=$X1&YMMTmakenames=$X2&YMMTmodelnames=$X3&YMMTtrimnames=$X4&YMMTsectionnames=$X5″URL_LIST_MD5=`echo”$URL_LIST”|md5sum|awk'{print$1}’`if!grep-q$URL_LIST_MD5″$URL_MD5″;then$CURL-s”$URL_LIST”>”$URL_MD5_DIR/$URL_LIST_MD5″NUM=`grep’Viewpage'”$URL_MD5_DIR/$URL_LIST_MD5″|wc-l`NUM2=$(($NUM/2))echo>$OPT5grep’ahref=”index.cfm?fuseaction=store.PartInfo&PartNumbe'”$URL_MD5_DIR/$URL_LIST_MD5″|cut-d'”‘-f2>$OPT5while[$NUM2-ge2];doURL_LIST=`grep”Viewpage$NUM2″”$URL_MD5_DIR/$URL_LIST_MD5″|awk-F'[“]”{a[$9]=$9}END{for(iina)printa[i]}’`$CURL-s”$URL_LIST”|grep’ahref=”index.cfm?fuseaction=store.PartInfo&PartNumbe’|cut-d'”‘-f2>>$OPT5NUM2=$(($NUM2-1))doneecho$URL_LIST_MD5>>”$URL_MD5″fiwhilereadX6;doURL_DETAIL=”${WEB_URL}${X6}”URL_DETAIL_MD=`echo$URL_DETAIL|md5sum|awk'{print$1}’`if!grep-q$URL_DETAIL_MD”$URL_MD5″>&/dev/null;then#该判断以商品列表详细页URL的md5值为基准，负责URL的重复项判定$CURL-s”$URL_DETAIL”>”$HTML_DIR/$URL_DETAIL_MD”LABEL=`grep’diagram-label'”$HTML_DIR/$URL_DETAIL_MD”|awk-F'[<>]”{print$5}’`#商品标签GIF_URL=`grep-B10partInfo”$HTML_DIR/$URL_DETAIL_MD”|grep-o”https.*gif”|awk'{a=$0}END{printa}’`#产品对应的图片URLPRODUCT_ID=`grep’productID'”$HTML_DIR/$URL_DETAIL_MD”|awk-F'[<>]”{print$3}’`#产品零件号码GIFILE=${GIF_URL#*/}#去除了https:/后的图片URL信息，as:/a/b.gifGIF_IMG=”${ROOT_DIR}${GIFILE}”#图片存到本地后的绝对路径，as:/dataimg/a/b.gifU4=`grep-B10′<!–startopentop–>'”$HTML_DIR/$URL_DETAIL_MD”|grepjavascript|awk-F'[<>]”{print$3}’`!ls$GIF_IMG>&/dev/null&&wget-q-m-k-P”$ROOT_DIR””$GIF_URL”echo$URL_DETAIL_MD>>”$URL_MD5″echo”$(date+%m%d%T)+++$X1+++$X2+++$X3+++$U4+++$X5+++$URL_DETAIL+++$URL_DETAIL_MD+++$LABEL+++$PRODUCT_ID+++$GIF_IMG+++$URL_LIST”>>”$REPORT”fidone<$OPT5#传入商品详细列表url信息，进行循环done<$OPT4#传入产品部位分类信息，进行循环done<$OPT3#传入装饰信息，，进行循环done<$OPT2#传入车型信息，进行循环done<$OPT1#传入品牌信息，进行循环echo>&9}&done<$OPT0#传入年份信息，进行循环waitexec9<&-
OK!
销售世界上第一号的产品–不是汽车，而是自己。
