浅谈python中爬虫框架（talonspider）的介绍

import time

from talonspider import Item, TextField, AttrField

from pprint import pprint

class TestSpider(Item):

  title = TextField(css_select='.book-info>h1>em')

  author = TextField(css_select='a.writer')

  cover = AttrField(css_select='a#bookImg>img', attr='src')

  def tal_title(self, title):

    return title

  def tal_cover(self, cover):

    return 'http:' + cover

if name == 'main':

  item_data = TestSpider.get_item(url='http://book.qidian.com/info/1004608738')

  pprint(item_data)

from talonspider import Item, TextField, AttrField

from pprint import pprint

# 定义继承自item的爬虫类

class DoubanSpider(Item):

  target_item = TextField(css_select='p.item')

  title = TextField(css_select='span.title')

  cover = AttrField(css_select='p.pic>a>img', attr='src')

  abstract = TextField(css_select='span.inq')

  def tal_title(self, title):

    if isinstance(title, str):

      return title

    else:

      return ''.join([i.text.strip().replace('\xa0', '') for i in title])

if name == 'main':

  items_data = DoubanSpider.get_items(url='movie.douban.com/top250')

  result = []

  for item in items_data:

    result.append({

      'title': item.title,

      'cover': item.cover,

      'abstract': item.abstract,

    })

  pprint(result)

# !/usr/bin/env python

from talonspider import Spider, Item, TextField, AttrField, Request

from talonspider.utils import get_random_user_agent

# 定义继承自item的爬虫类

class DoubanItem(Item):

  target_item = TextField(css_select='p.item')

  title = TextField(css_select='span.title')

  cover = AttrField(css_select='p.pic>a>img', attr='src')

  abstract = TextField(css_select='span.inq')

  def tal_title(self, title):

    if isinstance(title, str):

      return title

    else:

      return ''.join([i.text.strip().replace('\xa0', '') for i in title])

class DoubanSpider(Spider):

  # 定义起始url，必须

  start_urls = ['https://movie.douban.com/top250']

  # requests配置

  request_config = {

    'RETRIES': 3,

    'DELAY': 0,

    'TIMEOUT': 20

  }

  # 解析函数 必须有

  def parse(self, html):

    # 将html转化为etree

    etree = self.e_html(html)

    # 提取目标值生成新的url

    pages = [i.get('href') for i in etree.cssselect('.paginator>a')]

    pages.insert(0, '?start=0&filter=')

    headers = {

      "User-Agent": get_random_user_agent()

    }

    for page in pages:

      url = self.start_urls[0] + page

      yield Request(url, request_config=self.request_config, headers=headers, callback=self.parse_item)

  def parse_item(self, html):

    items_data = DoubanItem.get_items(html=html)

    # result = []

    for item in items_data:

      # result.append({

      #   'title': item.title,

      #   'cover': item.cover,

      #   'abstract': item.abstract,

      # })

      # 保存

      with open('douban250.txt', 'a+') as f:

        f.writelines(item.title + '\n')

if name == 'main':

  DoubanSpider.start()