Python爬虫爬取Google Play 100万个App的数据，并入库到数据库 scrapy框架

admin管理员组

文章数量:1604637

代码目录结构

相关文件代码

 google.py爬虫主要代码

  # -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors.sgml import SgmlLinkExtractor
from scrapy.linkextractors import LinkExtractor
from app.items import GoogleItem
from language_linkextractor import LanguageLinkExtractor
import urlparse
import sys

class GoogleSpider(CrawlSpider):
    reload(sys)
    sys.setdefaultencoding('utf-8')
    name = "google"
    allowed_domains = ["play.google"]
    start_urls = (
        'http://play.google/',
        'https://play.google/store/apps/details?id=me.ele'
    )
    rules = [
            Rule(LanguageLinkExtractor(allow=("/store/apps/details", )), callback='parse_app',follow=True),
        ] #  


    def parse_app(self, response):
        # 在这里只获取页面的 URL 以及下载数量
        item = GoogleItem()
        # item['url'] = response.url
        r = urlparse.urlparse(response.url);
        params = urlparse.parse_qs(r.query, True);
        item['package'] = ','.join(params['id']);
        item['num'] =  response.xpath("//div[@itemprop='numDownloads']").xpath("text()").extract()
        item['score'] =  response.xpath("//div[@class='score']").xpath("text()").extract()
        item['review'] =  response.xpath("//span[@class='reviews-num']").xpath("text()").extract()
        item['company'] =  response.xpath("//span[@itemprop='name']").xpath("text()").extract()
        item['category'] =  response.xpath("//span[@itemprop='genre']").xpath("text()").extract()
        item['name'] = response.xpath("//div[@class='id-app-title']").xpath("text()").extract() 
        yield item

language_linkextractor.py解决语言乱码问题

from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor


class LanguageLinkExtractor(LxmlLinkExtractor):
    # def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
    #              canonicalize=True,
    #              unique=True, process_value=None, deny_extensions=None, restrict_css=()):
    #     super(LxmlLinkExtractor, self).__init__(allow=allow, deny=deny,
    #         allow_domains=allow_domains, deny_domains=deny_domains,
    #         restrict_xpaths=restrict_xpaths, canonicalize=canonicalize,
    #         deny_extensions=deny_extensions, restrict_css=restrict_css)
    @staticmethod
    def addParams(url):
        if url.find('?') >= 0:
            return url+'&hl=en';
        else:
            return url +'?hl=en'; 


    def extract_links(self, response):
        links = LxmlLinkExtractor.extract_links(self, response);
        for x in links:
            x.url = LanguageLinkExtractor.addParams(x.url)
        # links = super(LxmlLinkExtractor, self).extract_links(response);
        return links;

items.py相关item

# -*- coding: utf-8 -*-


# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy/en/latest/topics/items.html

import scrapy
class GoogleItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # url = scrapy.Field()
    num = scrapy.Field()
    package = scrapy.Field();
    score = scrapy.Field();
    review = scrapy.Field();
    company = scrapy.Field();
    category = scrapy.Field();
    name = scrapy.Field();

pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy/en/latest/topics/item-pipeline.html

class AppPipeline(object):
    def process_item(self, item, spider):
        return item

setting.py

# -*- coding: utf-8 -*-


# Scrapy settings for app project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy/en/latest/topics/settings.html
#     http://scrapy.readthedocs/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs/en/latest/topics/spider-middleware.html


BOT_NAME = 'app'


SPIDER_MODULES = ['app.spiders']
NEWSPIDER_MODULE = 'app.spiders'




# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'app (+http://www.yourdomain)'


# Obey robots.txt rules
ROBOTSTXT_OBEY = True


# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32


# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16


# Disable cookies (enabled by default)
#COOKIES_ENABLED = False


# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False


# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}


# Enable or disable spider middlewares
# See http://scrapy.readthedocs/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'app.middlewares.MyCustomSpiderMiddleware': 543,
#}


# Enable or disable downloader middlewares
# See http://scrapy.readthedocs/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'app.middlewares.MyCustomDownloaderMiddleware': 543,
#}


# Enable or disable extensions
# See http://scrapy.readthedocs/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}


# Configure item pipelines
# See http://scrapy.readthedocs/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
#    'app.pipelines.SomePipeline': 300,
#}


# Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False


# Enable and configure HTTP caching (disabled by default)
# See http://scrapy.readthedocs/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'


# ITEM_PIPELINES = {
#   'scrapy_mongodb.MongoDBPipeline': 100
# }


# MONGODB_URI = 'mongodb://127.0.0.1:27017'
# MONGODB_DATABASE = 'scrapy'
# MONGODB_COLLECTION = 'play'
FEED_URL='google_play.csv'
FEED_FORMAT='csv'

scrapy.cfg

# Automatically created by: scrapy startproject
#
# For more information about the [deploy] section see:
# https://scrapyd.readthedocs/en/latest/deploy.html


[settings]
default = app.settings


[deploy]
#url = http://localhost:6800/
project = app

本文标签： 爬虫万个框架数据库数据