import requestsfrom bs4 import BeautifulSoup

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'  ' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}

start_url = 'https://www.pythonscraping.com'r = requests.get(start_url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')# 获取所有img标签img_tags = soup.find_all('img')for tag in img_tags:print(tag['src'])

import csvimport requestsfrom bs4 import BeautifulSoup

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'  ' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}

url = 'https://en.wikipedia.org/wiki/Comparison_of_text_editors'r = requests.get(url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')# 只要第一个表格rows = soup.find('table', class_='wikitable').find_all('tr')# csv写入时候每写一行会有一空行被写入，所以设置newline为空with open('editors.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f)for row in rows:
        csv_row = []for cell in row.find_all(['th', 'td']):
            csv_row.append(cell.text)

        writer.writerow(csv_row)

from io import StringIOimport csvimport requests

csv_data = requests.get('http://pythonscraping.com/files/MontyPythonAlbums.csv')
data_file = StringIO(csv_data.text)
reader = csv.reader(data_file)for row in reader:print(row)

from io import StringIOimport csvimport requests

csv_data = requests.get('http://pythonscraping.com/files/MontyPythonAlbums.csv')
data_file = StringIO(csv_data.text)
reader = csv.DictReader(data_file)# 查看Keyprint(reader.fieldnames)for row in reader:print(row['Year'], row['Name'], sep=': ')

import reimport pymysqlimport requestsfrom bs4 import BeautifulSoup

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'  ' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}

conn = pymysql.connect(host='localhost', user='root', password='admin', db='wiki', charset='utf8')
cur = conn.cursor()def insert_page_if_not_exists(url):
    cur.execute(f"SELECT * FROM pages WHERE url='{url}';")# 这条url没有插入的话if cur.rowcount == 0:# 那就插入cur.execute(f"INSERT INTO pages(url) VALUES('{url}');")
        conn.commit()# 刚插入数据的idreturn cur.lastrowid# 否则已经存在这条数据，因为url一般是唯一的，所以获取一个就行，取脚标0是获得idelse:return cur.fetchone()[0]def insert_link(from_page, to_page):print(from_page, ' -> ', to_page)
    cur.execute(f"SELECT * FROM links WHERE fromId={from_page} AND toId={to_page};")# 如果查询不到数据，则插入，插入需要两个pages的id，即两个urlif cur.rowcount == 0:
        cur.execute(f"INSERT INTO links(fromId, toId) VALUES({from_page}, {to_page});")
        conn.commit()# 链接去重pages = set()# 得到所有链接def get_links(page_url, recursion_level):global pagesif recursion_level == 0:return# 这是刚插入的链接page_id = insert_page_if_not_exists(page_url)
    r = requests.get('https://en.wikipedia.org' + page_url, headers=headers)
    soup = BeautifulSoup(r.text, 'lxml')
    link_tags = soup.find_all('a', href=re.compile('^/wiki/[^:/]*$'))for link_tag in link_tags:# page_id是刚插入的url，参数里再次调用了insert_page...方法，获得了刚插入的url里能去往的url列表# 由此形成联系，比如刚插入的id为1，id为1的url里能去往的id有2、3、4...，则形成1 -> 2, 1 -> 3这样的联系insert_link(page_id, insert_page_if_not_exists(link_tag['href']))if link_tag['href'] not in pages:
            new_page = link_tag['href']
            pages.add(new_page)# 递归查找, 只能递归recursion_level次get_links(new_page, recursion_level - 1)if __name__ == '__main__':try:
        get_links('/wiki/Kevin_Bacon', 5)except Exception as e:print(e)finally:
        cur.close()
        conn.close()