python 爬虫爬取腾讯新闻科技类的企鹅智酷系列（1）

废话不多说，直接贴代码，主要采用BeautifulSoup写的

# -*- coding: utf-8 -*-

"""

Created on Mon May 18 19:12:06 2015

@author: Administrator

"""

import urllib

import os

from bs4 import BeautifulSoup

import sys

reload(sys)

sys.setdefaultencoding("utf-8")

i = 0

j = 0

list_a = []

def gettext(href):

global j,list_a

page = urllib.urlopen(href).read()

soup = BeautifulSoup(page,from_encoding="gb18030")

div = soup.find_all("div",class_="content")

p_text = div[0].find_all("p")

for p in p_text:

fp = file("%s.txt" % list_a[j],"a")

fp.write(‘ ‘)

fp.write(p.get_text())

fp.write(" \n")

j+=1

def gethref(url): #获得所有链接

global i,list_a

fp = file("AllTitle.txt","w+")

page = urllib.urlopen(url).read()

soup = BeautifulSoup(page,from_encoding="gb18030")

ul = soup.find_all("ul",class_="row1")

li = ul[0].find_all("li")

for lia in li:

list_a.append(("%s、" % (i+1))+lia.h3.get_text())

href = lia.a.get(‘href‘)

# 将标题简介和链接有规则的写入文件中

fp.write("%s、" % (i+1))

i+=1

fp.write("标题：")

fp.write(lia.h3.get_text())

fp.write("\n 简介：")

fp.write(lia.p.get_text())

fp.write("\n 链接：")

fp.write(lia.a.get("href"))

fp.write("\n")

gettext(href)

if "__main__"==__name__:

url ="http://re.qq.com/biznext/zkht.htm"

gethref(url)

print "All Is OK!"