python如何示例爬虫代码

from urllib import request

req = request.urlopen("http://www.baidu.com")

print(req.read().decode("utf-8"))

req = request.Request(url) #此处url为某个网址

req.add_header(key,value)  #key即user-Agent，value即浏览器的版本信息

resp = request.urlopen(req)

print(resp.read().decode("utf-8"))

from urllib import parse

postData = parse.urlencode([

    (key1,val1),

    (key2,val2),

    (keyn,valn)

])

request.urlopen(req,data=postData.encode("utf-8")) #使用postData发送post请求

resp.status  #得到请求状态

resp.reason #得到服务器的类型

#-*- coding:utf-8 -*-

from bs4 import BeautifulSoup as bs

from urllib.request import urlopen

import re

import ssl

#获取维基百科词条信息

ssl._create_default_https_context = ssl._create_unverified_context #全局取消证书验证

#请求URL，并把结果用utf-8编码

req = urlopen("https://en.wikipedia.org/wiki/Main page").read().decode("utf-8")

#使用beautifulsoup去解析

soup = bs(req,"html.parser")

# print(soup)

#获取所有href属性以“/wiki/Special”开头的a标签

urllist = soup.findAll("a",href=re.compile("^/wiki/Special"))

for url in urllist:

#去除以.jpg或.JPG结尾的链接

if not re.search("\.(jpg|JPG)$",url["href"]):

#get_test()输出标签下的所有内容，包括子标签的内容；

#string只输出一个内容，若该标签有子标签则输出“none

print(url.get_text()+"----->"+url["href"])

# print(url)

$ pip install pymysql

$ python setup.py install

#引入开发包

import pymysql.cursors

#获取数据库链接

connection = pymysql.connect(host="localhost",

user = 'root',

password = '123456',

db ='wikiurl',

charset = 'utf8mb4')

try:

#获取会话指针

with connection.cursor() as cursor

#创建sql语句

sql = "insert into `tableName`(`urlname`,`urlhref`) values(%s,%s)"

#执行SQL语句

cursor.execute(sql,(url.get_text(),"https://en.wikipedia.org"+url["href"]))

#提交

connection.commit()

finally:

#关闭

connection.close()

Disallow：不允许访问

allow：允许访问