pytho2.7.x+requests+xpath爬取电影天堂

admin管理员组

文章数量:1657385

python2.7.x，和python3.x处理中文乱码方式不一样。遇到问题就百度，遇到python2的中文乱码问题可以参考这几篇博客：
https://blog.csdn/hp_cpp/article/details/80953508
https://blog.csdn/hp_cpp/article/details/80997208
https://blog.csdn/hp_cpp/article/details/84842534
https://blog.csdn/hp_cpp/article/details/80953251
本篇博客是自己看视频教程后的代码，参考原来的视频，可能是python版本的不同，我遇到的很多编码问题视频教程中并没有遇到，遇到问题就百度，记录一下，算是python初级入门系列中的一个吧。

爬取电影天堂前7页电影信息

#!/usr/bin/env python 
# -*- coding:utf-8 -*-

from lxml import etree
import requests

import sys
reload(sys)
sys.setdefaultencoding('utf-8')


BASE_DOMAIN = 'https://www.dytt8'
firs_url = "https://www.dytt8/html/gndy/dyzz/list_23_1.html"
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3610.2 Mobile Safari/537.36'
}


def get_detail_urls(url):
    response = requests.get(url, headers=HEADERS)
    text = response.text
    html = etree.HTML(text)
    detail_urls = html.xpath("//table[@class='tbspan']//a/@href")
    detail_urls = map(lambda url:  BASE_DOMAIN+url, detail_urls)
    return detail_urls


def parse_detail_page(url):
    movie = {}
    response = requests.get(url, headers=HEADERS)
    text = response.content.decode('gbk', 'ignore')
    html = etree.HTML(text)
    title = html.xpath("//div[@class='title_all']//font[@color='#07519a']/text()")[0]
    movie['title'] = title
    zoomE = html.xpath("//div[@id='Zoom']")[0]
    imgs = zoomE.xpath(".//img")
    #print(title, url)
    cover = imgs[0]
    movie['cover'] = cover
    if len(imgs) >= 2:
        screenshot = imgs[1]
        movie['screenshot'] = screenshot

    def parse_info(info, rule):
        return info.replace(rule, "").strip()

    infos = zoomE.xpath(".//text()")
    for index, info in enumerate(infos):
        if info.startswith("◎年　　代"):
            info = parse_info(info, "◎年　　代")
            movie['year'] = info
        elif info.startswith("◎产　　地"):
            info = parse_info(info, "◎产　　地")
            movie['country'] = info
        elif info.startswith("◎类　　别"):
            info = parse_info(info, "◎类　　别")
            movie['category'] = info
        elif info.startswith("◎豆瓣评分"):
            info = parse_info(info, "◎豆瓣评分")
            movie['douban_rating'] = info
        elif info.startswith("◎片　　长"):
            info = parse_info(info, "◎片　　长")
            movie['duration'] = info
        elif info.startswith("◎导　　演"):
            info = parse_info(info, "◎导　　演")
            movie['director'] = info
        elif info.startswith("◎主　　演"):
            info = parse_info(info, "◎主　　演")
            actors = [info]
            for x in range(index+1, len(infos)):
                actor = infos[x].strip()
                #actor = actor.encode('gbk', 'ignore')
                if actor.startswith("◎"):
                    break
                actors.append(actor)
            #for actor in actors:
            #    print(actor.encode('gbk', 'ignore'))
        elif info.startswith("◎简　　介 "):
            info = parse_info(info, "◎简　　介 ")  #电影简介多段的情况
            profiles = []
            for x in range(index+1, len(infos)):
                profile = infos[x].strip()
                if profile.startswith("【下载地址】") or profile.startswith("◎"):
                    break
                else:
                    profiles.append(profile)
            movie['profile'] = ''.join(profiles)
            print(url)
            print(movie['profile'].encode('utf-8'))
    download_url = html.xpath("//td[@bgcolor='#fdfddf']/a/@href")[0]
    movie['download_url'] = download_url
    return movie

def spider():
    base_url = "https://www.dytt8/html/gndy/dyzz/list_23_{}.html"
    movies = []
    for x in range(1, 8):
        url = base_url.format(x)
        detal_urls = get_detail_urls(url)
        for detal_url in detal_urls:
            movie = parse_detail_page(detal_url)
            movies.append(movie)
            #print(movie)

if __name__ == '__main__':
    spider()

遇到的问题

编码问题

（1）会出现中文乱码的情况，设置了# -- coding:utf-8 --和

reload(sys)
sys.setdefaultencoding('utf-8')

再加上明白当前字符串是采用的什么编码方式，一般可以解决。
（2）打印的时候出现一些UnicodeEncodeError的问题，具体参考本博客开头的链接。

数据格式不规范的问题

例如在https://www.dytt8/html/gndy/dyzz/20181008/57588.html’
这个页面，是没有电影海报图片，但是有电影封面图片。简介中有多段文字的情况。还有简介文字分了几段的情况，还有◎获奖情况 ，有的电影详细页面没有这个，所以要特殊处理。

本文标签： 天堂电影XPathrequests