因为看见老男孩的学员做过这一期，也很有兴趣，并且刚在学python，所以用python实现了下。由于刚学习python，肯定还有很多地方可以优化。
#!/bin/bash
# oldboy linux training
# 2015-06-01
# Happy Children‘s Day
# 说明：本脚本来自老男孩linux21期学员张耀开发！
EduFile=/tmp/edu.html
EduFile2=/tmp/edu2.html
Url="$*"
# Check for given parameters
[ $# -eq 0 ] && {
echo "USAGE: /bin/sh $0 http://...."
exit 1
}
# Judge url is ok?
curl -I $Url &>/dev/null
[ $? -ne 0 ] &&{
echo "Bad url,Please check it"
exit 1
}
# Defined get pagenum and CourseId Functions
function getnum(){
curl -s $Url>$EduFile
grep ‘"pagesGoEnd"‘ $EduFile &>/dev/null
if [ $? -eq 0 ]
then
num=`sed -rn ‘s#.*page=([0-9].*)" class="pagesGoEnd".*$#\1#gp‘ $EduFile`
else
num=`sed -rn ‘s|.*page=([0-9].*)#" class="pagesNum".*$|\1|gp‘ $EduFile`
fi
pagenum=${num:-1}
CourseId=`echo $Url|awk -F "[-.]" ‘{print $4}‘`
}
# Defined curl html Functions
function Curl(){
getnum
for i in `seq $pagenum`
do
curl "http://edu.51cto.com/index.php?do=course&m=lessions&course_id=$CourseId&page=$i" 1>>$EduFile 2>/dev/null
done
}
# Defined Create table Functions
function table(){
sum=""
index=1
sed -rn ‘/do=lesson/ s#<.*(<a href=")(.*)</h4>$#\1http://edu.51cto.com\2#gp‘ $EduFile > $EduFile2
while read line
do
sum=$sum"<tr><th width="40" scope="row">$index</th><td width="520">$line</td>"
((index++))
done <$EduFile2
}
# Defined Create html Functions
function html(){
cat >/tmp/oldboy.html<<-END
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>test</title>
</head>
<body>
<table width="560" border="1">
$sum
</table>
</body>
</html>
END
}
function main(){
Curl
table
html
}
main
#!/usr/bin/env python #coding:utf-8 import urllib,urllib2,sys,os,re reload(sys) sys.setdefaultencoding(‘utf-8‘) def get_course_id_page_num(URI): orgin = urllib2.urlopen(URI).read().decode(‘utf-8‘).encode(‘utf-8‘) try: page_pattern = re.compile(r‘<a href="/index.php?.*course_id=(.*)&page=(.*)" class="pagesGoEnd"‘) course_id = page_pattern.search(orgin).group(1) pagenum = page_pattern.search(orgin).group(2) except AttributeError: page_pattern = re.compile(r‘<a href="/index.php?.*course_id=(.*)&page=(.*)#" class="pagesNum"‘) course_id = page_pattern.search(orgin).group(1) pagenum = page_pattern.search(orgin).group(2) return (course_id,pagenum) def get_url_title(course,page): #cto = file(‘/tmp/edu.51cto.html‘,‘a+‘) url= "http://edu.51cto.com/index.php?do=course&m=lessions&course_id=" + str(course) + "&page=" + str(page) request = urllib2.urlopen(url).read().decode(‘utf-8‘).encode(‘utf-8‘) url_title_pattern = re.compile(r‘<a href="(.*)" target="_blank">(.*)</a></h4>‘) ut=url_title_pattern.findall(request) cto_table_1 = ‘<tr><th scope=row>+_+</th><td><a href="‘ cto_table_2 = ‘http://edu.51cto.com‘ cto_table_3 = ‘" target="_blank">‘ cto_table_4 = ‘</a></td>‘ for k,v in ut: if ‘lesson‘ in k: line = cto_table_1 + cto_table_2 + k + cto_table_3 + v + cto_table_4 print >> cto, line #cto.close() if __name__ == "__main__": URI = "http://edu.51cto.com/course/course_id-839.html" #URI = sys.argv[1] cto = file(‘/tmp/edu.51cto.html‘,‘a+‘) cto_head = ‘<head>\n<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>\n<title>51cto</title>\n</head>\n<body><table width="560" border="1">\n‘ print >> cto, cto_head course_id,pagenum = get_course_id_page_num(URI) for i in range(1,int(pagenum) + 1): get_url_title(course_id,i) cto_tail = ‘</table>\n</body>\n</html>‘ print >> cto, cto_tail cto.close()
以下就是程序运行之后的效果图了
时间： 2024-10-21 15:11:24