class MysqlTwistedPipline(object):
def __init__(self,dbpool):
self.dbpool = dbpool @classmethod
def from_settings(cls,settings):
dbparms = dict(
host = settings["MYSQL_HOST"],
db = settings["MYSQL_DBNAME"],
user = settings["MYSQL_USER"],
passwd = settings["MYSQL_PASSWORD"],
charset = 'utf8',
cursorclass = MySQLdb.cursors.DictCursor,
use_unicode=True,
)
dbpool = adbapi.ConnectionPool("MySQLdb",**dbparms) return cls(dbpool) def process_item(self, item, spider):
#使用Twisted将mysql插入变成异步执行
#runInteraction可以将传入的函数变成异步的
query = self.dbpool.runInteraction(self.do_insert,item) #处理异常
query.addErrback(self.handle_error,item,spider) def handle_error(self,failure,item,spider):
#处理异步插入的异常
print(failure) def do_insert(self,cursor,item):
#会从dbpool取出cursor
#执行具体的插入
insert_sql = """
insert into jobbole_article(title,create_date,url,url_object_id) VALUES (%s,%s,%s,%s)
"""
cursor.execute(insert_sql, (item["title"], item["create_date"], item["url"], item["url_object_id"])) #拿传进的cursor进行执行，并且自动完成commit操作
