mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into enhancement/alternate-feedexport-constructors
This commit is contained in:
Eugenio Lacuesta
commit
917c1fde68
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 1.4.0
|
||||
current_version = 1.5.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
11
.coveragerc
11
.coveragerc
|
|
@ -5,15 +5,4 @@ omit =
|
|||
tests/*
|
||||
scrapy/xlib/*
|
||||
scrapy/conf.py
|
||||
scrapy/stats.py
|
||||
scrapy/project.py
|
||||
scrapy/utils/decorator.py
|
||||
scrapy/statscol.py
|
||||
scrapy/squeue.py
|
||||
scrapy/log.py
|
||||
scrapy/dupefilter.py
|
||||
scrapy/command.py
|
||||
scrapy/linkextractor.py
|
||||
scrapy/spider.py
|
||||
scrapy/contrib/*
|
||||
scrapy/contrib_exp/*
|
||||
|
|
|
|||
31
.travis.yml
31
.travis.yml
|
|
@ -13,27 +13,34 @@ matrix:
|
|||
env: TOXENV=jessie
|
||||
- python: 2.7
|
||||
env: TOXENV=pypy
|
||||
- python: 3.3
|
||||
env: TOXENV=py33
|
||||
- python: 2.7
|
||||
env: TOXENV=pypy3
|
||||
- python: 3.4
|
||||
env: TOXENV=py34
|
||||
- python: 3.5
|
||||
env: TOXENV=py35
|
||||
- python: 3.6
|
||||
env: TOXENV=py36
|
||||
- python: 3.7
|
||||
env: TOXENV=py37
|
||||
dist: xenial
|
||||
sudo: true
|
||||
- python: 3.6
|
||||
env: TOXENV=docs
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy" ]; then
|
||||
export PYENV_ROOT="$HOME/.pyenv"
|
||||
if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
|
||||
pushd "$PYENV_ROOT" && git pull && popd
|
||||
else
|
||||
rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
|
||||
fi
|
||||
# get latest portable PyPy from pyenv directly (thanks to natural version sort option -V)
|
||||
export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-portable-[0-9][\.0-9]*$' |sort -V |tail -1`
|
||||
"$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION"
|
||||
virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
- pip install -U tox twine wheel codecov
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as
|
||||
contributors and maintainers pledge to making participation in our project and
|
||||
contributors and maintainers pledge to make participation in our project and
|
||||
our community a harassment-free experience for everyone, regardless of age, body
|
||||
size, disability, ethnicity, gender identity and expression, level of experience,
|
||||
nationality, personal appearance, race, religion, or sexual identity and
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
The guidelines for contributing are available here:
|
||||
http://doc.scrapy.org/en/master/contributing.html
|
||||
https://doc.scrapy.org/en/master/contributing.html
|
||||
|
||||
Please do not abuse the issue tracker for support questions.
|
||||
If your issue topic can be rephrased to "How to ...?", please use the
|
||||
support channels to get it answered: http://scrapy.org/community/
|
||||
support channels to get it answered: https://scrapy.org/community/
|
||||
|
|
|
|||
2
INSTALL
2
INSTALL
|
|
@ -1,4 +1,4 @@
|
|||
For information about installing Scrapy see:
|
||||
|
||||
* docs/intro/install.rst (local file)
|
||||
* http://doc.scrapy.org/en/latest/intro/install.html (online version)
|
||||
* https://doc.scrapy.org/en/latest/intro/install.html (online version)
|
||||
|
|
|
|||
4
LICENSE
4
LICENSE
|
|
@ -5,10 +5,10 @@ Redistribution and use in source and binary forms, with or without modification,
|
|||
are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
this list of conditions, and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
notice, this list of conditions, and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of Scrapy nor the names of its contributors may be used
|
||||
|
|
|
|||
24
README.rst
24
README.rst
|
|
@ -6,8 +6,12 @@ Scrapy
|
|||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: PyPI Version
|
||||
|
||||
.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg
|
||||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: Supported Python Versions
|
||||
|
||||
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
|
||||
:target: http://travis-ci.org/scrapy/scrapy
|
||||
:target: https://travis-ci.org/scrapy/scrapy
|
||||
:alt: Build Status
|
||||
|
||||
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
|
||||
|
|
@ -15,7 +19,7 @@ Scrapy
|
|||
:alt: Wheel Status
|
||||
|
||||
.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
|
||||
:target: http://codecov.io/github/scrapy/scrapy?branch=master
|
||||
:target: https://codecov.io/github/scrapy/scrapy?branch=master
|
||||
:alt: Coverage report
|
||||
|
||||
.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg
|
||||
|
|
@ -31,12 +35,12 @@ crawl websites and extract structured data from their pages. It can be used for
|
|||
a wide range of purposes, from data mining to monitoring and automated testing.
|
||||
|
||||
For more information including a list of features check the Scrapy homepage at:
|
||||
http://scrapy.org
|
||||
https://scrapy.org
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
* Python 2.7 or Python 3.3+
|
||||
* Python 2.7 or Python 3.4+
|
||||
* Works on Linux, Windows, Mac OSX, BSD
|
||||
|
||||
Install
|
||||
|
|
@ -47,12 +51,12 @@ The quick way::
|
|||
pip install scrapy
|
||||
|
||||
For more details see the install section in the documentation:
|
||||
http://doc.scrapy.org/en/latest/intro/install.html
|
||||
https://doc.scrapy.org/en/latest/intro/install.html
|
||||
|
||||
Documentation
|
||||
=============
|
||||
|
||||
Documentation is available online at http://doc.scrapy.org/ and in the ``docs``
|
||||
Documentation is available online at https://doc.scrapy.org/ and in the ``docs``
|
||||
directory.
|
||||
|
||||
Releases
|
||||
|
|
@ -63,12 +67,12 @@ You can find release notes at https://doc.scrapy.org/en/latest/news.html
|
|||
Community (blog, twitter, mail list, IRC)
|
||||
=========================================
|
||||
|
||||
See http://scrapy.org/community/
|
||||
See https://scrapy.org/community/
|
||||
|
||||
Contributing
|
||||
============
|
||||
|
||||
See http://doc.scrapy.org/en/master/contributing.html
|
||||
See https://doc.scrapy.org/en/master/contributing.html
|
||||
|
||||
Code of Conduct
|
||||
---------------
|
||||
|
|
@ -82,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
|
|||
Companies using Scrapy
|
||||
======================
|
||||
|
||||
See http://scrapy.org/companies/
|
||||
See https://scrapy.org/companies/
|
||||
|
||||
Commercial Support
|
||||
==================
|
||||
|
||||
See http://scrapy.org/support/
|
||||
See https://scrapy.org/support/
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ scrapy-logo.jpg
|
|||
|
||||
Main Scrapy logo, in JPEG format.
|
||||
|
||||
qlassik.zip
|
||||
qlassik.zip
|
||||
-----------
|
||||
|
||||
Font used for Scrapy logo. Homepage: http://www.dafont.com/qlassik.font
|
||||
Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
|
||||
|
||||
scrapy-blog.logo.xcf
|
||||
--------------------
|
||||
|
|
|
|||
11
conftest.py
11
conftest.py
|
|
@ -11,21 +11,12 @@ def _py_files(folder):
|
|||
collect_ignore = [
|
||||
# deprecated or moved modules
|
||||
"scrapy/conf.py",
|
||||
"scrapy/stats.py",
|
||||
"scrapy/project.py",
|
||||
"scrapy/utils/decorator.py",
|
||||
"scrapy/statscol.py",
|
||||
"scrapy/squeue.py",
|
||||
"scrapy/log.py",
|
||||
"scrapy/dupefilter.py",
|
||||
"scrapy/command.py",
|
||||
"scrapy/linkextractor.py",
|
||||
"scrapy/spider.py",
|
||||
|
||||
# not a test, but looks like a test
|
||||
"scrapy/utils/testsite.py",
|
||||
|
||||
] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
|
||||
]
|
||||
|
||||
if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
|
||||
collect_ignore += _py_files("scrapy/xlib/tx")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Priority: optional
|
|||
Maintainer: Scrapinghub Team <info@scrapinghub.com>
|
||||
Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
|
||||
Standards-Version: 3.8.4
|
||||
Homepage: http://scrapy.org/
|
||||
Homepage: https://scrapy.org/
|
||||
|
||||
Package: scrapy
|
||||
Architecture: all
|
||||
|
|
@ -15,6 +15,6 @@ Conflicts: python-scrapy, scrapy-0.25
|
|||
Provides: python-scrapy, scrapy-0.25
|
||||
Description: Python web crawling and web scraping framework
|
||||
Scrapy is a fast high-level web crawling and web scraping framework,
|
||||
used to crawl websites and extract structured data from their pages.
|
||||
It can be used for a wide range of purposes, from data mining to
|
||||
used to crawl websites and extract structured data from their pages.
|
||||
It can be used for a wide range of purposes, from data mining to
|
||||
monitoring and automated testing.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
|
||||
|
||||
It was downloaded from http://scrapy.org
|
||||
It was downloaded from https://scrapy.org
|
||||
|
||||
Upstream Author: Scrapy Developers
|
||||
|
||||
|
|
@ -14,10 +14,10 @@ All rights reserved.
|
|||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from docutils.parsers.rst.roles import set_classes
|
||||
from docutils import nodes
|
||||
from sphinx.util.compat import Directive
|
||||
from docutils.parsers.rst import Directive
|
||||
from sphinx.util.nodes import make_refnode
|
||||
from operator import itemgetter
|
||||
|
||||
|
|
@ -110,24 +110,28 @@ def setup(app):
|
|||
app.connect('doctree-read', collect_scrapy_settings_refs)
|
||||
app.connect('doctree-resolved', replace_settingslist_nodes)
|
||||
|
||||
|
||||
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
|
||||
def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'https://github.com/scrapy/scrapy/issues/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
|
||||
def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'https://github.com/scrapy/scrapy/commit/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
|
||||
def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
|
||||
set_classes(options)
|
||||
|
|
|
|||
|
|
@ -144,10 +144,6 @@ html_static_path = ['_static']
|
|||
# using the given strftime format.
|
||||
html_last_updated_fmt = '%b %d, %Y'
|
||||
|
||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||
# typographically correct entities.
|
||||
html_use_smartypants = True
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
#html_sidebars = {}
|
||||
|
||||
|
|
@ -191,8 +187,8 @@ htmlhelp_basename = 'Scrapydoc'
|
|||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, document class [howto/manual]).
|
||||
latex_documents = [
|
||||
('index', 'Scrapy.tex', ur'Scrapy Documentation',
|
||||
ur'Scrapy developers', 'manual'),
|
||||
('index', 'Scrapy.tex', u'Scrapy Documentation',
|
||||
u'Scrapy developers', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ Contributing to Scrapy
|
|||
|
||||
.. important::
|
||||
|
||||
Double check you are reading the most recent version of this document at
|
||||
http://doc.scrapy.org/en/master/contributing.html
|
||||
Double check that you are reading the most recent version of this document at
|
||||
https://doc.scrapy.org/en/master/contributing.html
|
||||
|
||||
There are many ways to contribute to Scrapy. Here are some of them:
|
||||
|
||||
|
|
@ -18,13 +18,17 @@ There are many ways to contribute to Scrapy. Here are some of them:
|
|||
* Report bugs and request features in the `issue tracker`_, trying to follow
|
||||
the guidelines detailed in `Reporting bugs`_ below.
|
||||
|
||||
* Submit patches for new functionality and/or bug fixes. Please read
|
||||
`Writing patches`_ and `Submitting patches`_ below for details on how to
|
||||
* Submit patches for new functionalities and/or bug fixes. Please read
|
||||
:ref:`writing-patches` and `Submitting patches`_ below for details on how to
|
||||
write and submit a patch.
|
||||
|
||||
* Join the `Scrapy subreddit`_ and share your ideas on how to
|
||||
improve Scrapy. We're always open to suggestions.
|
||||
|
||||
* Answer Scrapy questions at
|
||||
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__.
|
||||
|
||||
|
||||
Reporting bugs
|
||||
==============
|
||||
|
||||
|
|
@ -40,9 +44,14 @@ guidelines when reporting a new bug.
|
|||
* check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
|
||||
well-known question
|
||||
|
||||
* check the `open issues`_ to see if it has already been reported. If it has,
|
||||
don't dismiss the report but check the ticket history and comments, you may
|
||||
find additional useful information to contribute.
|
||||
* if you have a general question about scrapy usage, please ask it at
|
||||
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
|
||||
(use "scrapy" tag).
|
||||
|
||||
* check the `open issues`_ to see if the issue has already been reported. If it
|
||||
has, don't dismiss the report, but check the ticket history and comments. If
|
||||
you have additional useful information, please leave a comment, or consider
|
||||
:ref:`sending a pull request <writing-patches>` with a fix.
|
||||
|
||||
* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
|
||||
been discussed there, or if you're not sure if what you're seeing is a bug.
|
||||
|
|
@ -54,17 +63,24 @@ guidelines when reporting a new bug.
|
|||
it. See for example StackOverflow's guide on creating a
|
||||
`Minimal, Complete, and Verifiable example`_ exhibiting the issue.
|
||||
|
||||
* the most awesome way to provide a complete reproducible example is to
|
||||
send a pull request which adds a failing test case to the
|
||||
Scrapy testing suite (see :ref:`submitting-patches`).
|
||||
This is helpful even if you don't have an intention to
|
||||
fix the issue yourselves.
|
||||
|
||||
* include the output of ``scrapy version -v`` so developers working on your bug
|
||||
know exactly which version and platform it occurred on, which is often very
|
||||
helpful for reproducing it, or knowing if it was already fixed.
|
||||
|
||||
.. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve
|
||||
|
||||
.. _writing-patches:
|
||||
|
||||
Writing patches
|
||||
===============
|
||||
|
||||
The better written a patch is, the higher chance that it'll get accepted and
|
||||
the sooner that will be merged.
|
||||
The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged.
|
||||
|
||||
Well-written patches should:
|
||||
|
||||
|
|
@ -83,6 +99,8 @@ Well-written patches should:
|
|||
the documentation changes in the same patch. See `Documentation policies`_
|
||||
below.
|
||||
|
||||
.. _submitting-patches:
|
||||
|
||||
Submitting patches
|
||||
==================
|
||||
|
||||
|
|
@ -100,14 +118,32 @@ starting point is to send a pull request on GitHub. It can be simple enough to
|
|||
illustrate your idea, and leave documentation/tests for later, after the idea
|
||||
has been validated and proven useful. Alternatively, you can start a
|
||||
conversation in the `Scrapy subreddit`_ to discuss your idea first.
|
||||
|
||||
Sometimes there is an existing pull request for the problem you'd like to
|
||||
solve, which is stalled for some reason. Often the pull request is in a
|
||||
right direction, but changes are requested by Scrapy maintainers, and the
|
||||
original pull request author hasn't had time to address them.
|
||||
In this case consider picking up this pull request: open
|
||||
a new pull request with all commits from the original pull request, as well as
|
||||
additional changes to address the raised issues. Doing so helps a lot; it is
|
||||
not considered rude as soon as the original author is acknowledged by keeping
|
||||
his/her commits.
|
||||
|
||||
You can pull an existing pull request to a local branch
|
||||
by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE``
|
||||
(replace 'upstream' with a remote name for scrapy repository,
|
||||
``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE``
|
||||
with a name of the branch you want to create locally).
|
||||
See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally.
|
||||
|
||||
When writing GitHub pull requests, try to keep titles short but descriptive.
|
||||
E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
|
||||
prefer "Fix hanging when exception occurs in start_requests (#411)"
|
||||
instead of "Fix for #411".
|
||||
Complete titles make it easy to skim through the issue tracker.
|
||||
instead of "Fix for #411". Complete titles make it easy to skim through
|
||||
the issue tracker.
|
||||
|
||||
Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
|
||||
removal, etc) in separate commits than functional changes. This will make pull
|
||||
removal, etc) in separate commits from functional changes. This will make pull
|
||||
requests easier to review and more likely to get merged.
|
||||
|
||||
Coding style
|
||||
|
|
@ -121,21 +157,29 @@ Scrapy:
|
|||
* It's OK to use lines longer than 80 chars if it improves the code
|
||||
readability.
|
||||
|
||||
* Don't put your name in the code you contribute. Our policy is to keep
|
||||
the contributor's name in the `AUTHORS`_ file distributed with Scrapy.
|
||||
* Don't put your name in the code you contribute; git provides enough
|
||||
metadata to identify author of the code.
|
||||
See https://help.github.com/articles/setting-your-username-in-git/ for
|
||||
setup instructions.
|
||||
|
||||
Documentation policies
|
||||
======================
|
||||
|
||||
* **Don't** use docstrings for documenting classes, or methods which are
|
||||
already documented in the official (sphinx) documentation. For example, the
|
||||
:meth:`ItemLoader.add_value` method should be documented in the sphinx
|
||||
documentation, not its docstring.
|
||||
already documented in the official (sphinx) documentation. Alternatively,
|
||||
**do** provide a docstring, but make sure sphinx documentation uses
|
||||
autodoc_ extension to pull the docstring. For example, the
|
||||
:meth:`ItemLoader.add_value` method should be either
|
||||
documented only in the sphinx documentation (not as a docstring), or
|
||||
it should have a docstring which is pulled to sphinx documentation using
|
||||
autodoc_ extension.
|
||||
|
||||
* **Do** use docstrings for documenting functions not present in the official
|
||||
(sphinx) documentation, such as functions from ``scrapy.utils`` package and
|
||||
its sub-modules.
|
||||
|
||||
.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
|
||||
|
||||
Tests
|
||||
=====
|
||||
|
||||
|
|
@ -188,7 +232,7 @@ And their unit-tests are in::
|
|||
|
||||
.. _issue tracker: https://github.com/scrapy/scrapy/issues
|
||||
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
|
||||
.. _Scrapy subreddit: http://reddit.com/r/scrapy
|
||||
.. _Scrapy subreddit: https://reddit.com/r/scrapy
|
||||
.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
|
||||
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
|
||||
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ Python code.
|
|||
In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like
|
||||
comparing `jinja2`_ to `Django`_.
|
||||
|
||||
.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
|
||||
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
|
||||
.. _lxml: http://lxml.de/
|
||||
.. _jinja2: http://jinja.pocoo.org/
|
||||
.. _Django: https://www.djangoproject.com/
|
||||
|
|
@ -69,9 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
|
|||
What Python versions does Scrapy support?
|
||||
-----------------------------------------
|
||||
|
||||
Scrapy is supported under Python 2.7 and Python 3.3+.
|
||||
Scrapy is supported under Python 2.7 and Python 3.4+
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
Python 2.6 support was dropped starting at Scrapy 0.20.
|
||||
Python 3 support was added in Scrapy 1.1.
|
||||
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
|
||||
|
||||
.. note::
|
||||
For Python 3 support on Windows, it is recommended to use
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ Installation guide
|
|||
Installing Scrapy
|
||||
=================
|
||||
|
||||
Scrapy runs on Python 2.7 and Python 3.3 or above.
|
||||
Scrapy runs on Python 2.7 and Python 3.4 or above
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
|
||||
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
|
||||
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
|
||||
|
|
@ -132,12 +133,12 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
|
|||
|
||||
.. _intro-install-ubuntu:
|
||||
|
||||
Ubuntu 12.04 or above
|
||||
Ubuntu 14.04 or above
|
||||
---------------------
|
||||
|
||||
Scrapy is currently tested with recent-enough versions of lxml,
|
||||
twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions.
|
||||
But it should support older versions of Ubuntu too, like Ubuntu 12.04,
|
||||
But it should support older versions of Ubuntu too, like Ubuntu 14.04,
|
||||
albeit with potential issues with TLS connections.
|
||||
|
||||
**Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
|
||||
|
|
@ -163,8 +164,8 @@ you can install Scrapy with ``pip`` after that::
|
|||
pip install scrapy
|
||||
|
||||
.. note::
|
||||
The same non-python dependencies can be used to install Scrapy in Debian
|
||||
Wheezy (7.0) and above.
|
||||
The same non-Python dependencies can be used to install Scrapy in Debian
|
||||
Jessie (8.0) and above.
|
||||
|
||||
|
||||
.. _intro-install-macos:
|
||||
|
|
@ -188,7 +189,7 @@ solutions:
|
|||
that doesn't conflict with the rest of your system. Here's how to do it using
|
||||
the `homebrew`_ package manager:
|
||||
|
||||
* Install `homebrew`_ following the instructions in http://brew.sh/
|
||||
* Install `homebrew`_ following the instructions in https://brew.sh/
|
||||
|
||||
* Update your ``PATH`` variable to state that homebrew packages should be
|
||||
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
|
||||
|
|
@ -223,6 +224,29 @@ After any of these workarounds you should be able to install Scrapy::
|
|||
pip install Scrapy
|
||||
|
||||
|
||||
PyPy
|
||||
----
|
||||
|
||||
We recommend using the latest PyPy version. The version tested is 5.9.0.
|
||||
For PyPy3, only Linux installation was tested.
|
||||
|
||||
Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
|
||||
This means that these dependecies will be built during installation.
|
||||
On OS X, you are likely to face an issue with building Cryptography dependency,
|
||||
solution to this problem is described
|
||||
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
|
||||
that is to ``brew install openssl`` and then export the flags that this command
|
||||
recommends (only needed when installing scrapy). Installing on Linux has no special
|
||||
issues besides installing build dependencies.
|
||||
Installing scrapy with PyPy on Windows is not tested.
|
||||
|
||||
You can check that scrapy is installed correctly by running ``scrapy bench``.
|
||||
If this command gives errors such as
|
||||
``TypeError: ... got 2 unexpected keyword arguments``, this means
|
||||
that setuptools was unable to pick up one PyPy-specific dependency.
|
||||
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
|
||||
|
||||
|
||||
.. _Python: https://www.python.org/
|
||||
.. _pip: https://pip.pypa.io/en/latest/installing/
|
||||
.. _lxml: http://lxml.de/
|
||||
|
|
@ -233,9 +257,9 @@ After any of these workarounds you should be able to install Scrapy::
|
|||
.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
|
||||
.. _setuptools: https://pypi.python.org/pypi/setuptools
|
||||
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
|
||||
.. _homebrew: http://brew.sh/
|
||||
.. _zsh: http://www.zsh.org/
|
||||
.. _Scrapinghub: http://scrapinghub.com
|
||||
.. _Anaconda: http://docs.continuum.io/anaconda/index
|
||||
.. _Miniconda: http://conda.pydata.org/docs/install/quick.html
|
||||
.. _conda-forge: https://conda-forge.github.io/
|
||||
.. _homebrew: https://brew.sh/
|
||||
.. _zsh: https://www.zsh.org/
|
||||
.. _Scrapinghub: https://scrapinghub.com
|
||||
.. _Anaconda: https://docs.anaconda.com/anaconda/
|
||||
.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
|
||||
.. _conda-forge: https://conda-forge.org/
|
||||
|
|
|
|||
|
|
@ -160,8 +160,8 @@ The next steps for you are to :ref:`install Scrapy <intro-install>`,
|
|||
a full-blown Scrapy project and `join the community`_. Thanks for your
|
||||
interest!
|
||||
|
||||
.. _join the community: http://scrapy.org/community/
|
||||
.. _join the community: https://scrapy.org/community/
|
||||
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
|
||||
.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _Sitemaps: http://www.sitemaps.org
|
||||
.. _Sitemaps: https://www.sitemaps.org/index.html
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ list of Python resources for non-programmers`_.
|
|||
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
|
||||
.. _Dive Into Python 3: http://www.diveintopython3.net
|
||||
.. _Python Tutorial: https://docs.python.org/3/tutorial
|
||||
.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/
|
||||
.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/
|
||||
|
||||
|
||||
Creating a project
|
||||
|
|
@ -54,6 +54,8 @@ This will create a ``tutorial`` directory with the following contents::
|
|||
__init__.py
|
||||
|
||||
items.py # project items definition file
|
||||
|
||||
middlewares.py # project middlewares file
|
||||
|
||||
pipelines.py # project pipelines file
|
||||
|
||||
|
|
@ -452,7 +454,7 @@ For historic reasons, Scrapy appends to a given file instead of overwriting
|
|||
its contents. If you run this command twice without removing the file
|
||||
before the second time, you'll end up with a broken JSON file.
|
||||
|
||||
You can also used other formats, like `JSON Lines`_::
|
||||
You can also use other formats, like `JSON Lines`_::
|
||||
|
||||
scrapy crawl quotes -o quotes.jl
|
||||
|
||||
|
|
|
|||
139
docs/news.rst
139
docs/news.rst
|
|
@ -3,6 +3,143 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
Scrapy 1.6.0 (unreleased)
|
||||
-------------------------
|
||||
|
||||
Cleanups
|
||||
~~~~~~~~
|
||||
|
||||
* Remove deprecated ``CrawlerSettings`` class.
|
||||
* Remove deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes.
|
||||
|
||||
|
||||
Scrapy 1.5.1 (2018-07-12)
|
||||
-------------------------
|
||||
|
||||
This is a maintenance release with important bug fixes, but no new features:
|
||||
|
||||
* ``O(N^2)`` gzip decompression issue which affected Python 3 and PyPy
|
||||
is fixed (:issue:`3281`);
|
||||
* skipping of TLS validation errors is improved (:issue:`3166`);
|
||||
* Ctrl-C handling is fixed in Python 3.5+ (:issue:`3096`);
|
||||
* testing fixes (:issue:`3092`, :issue:`3263`);
|
||||
* documentation improvements (:issue:`3058`, :issue:`3059`, :issue:`3089`,
|
||||
:issue:`3123`, :issue:`3127`, :issue:`3189`, :issue:`3224`, :issue:`3280`,
|
||||
:issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`,
|
||||
:issue:`3294`).
|
||||
|
||||
Scrapy 1.5.0 (2017-12-29)
|
||||
-------------------------
|
||||
|
||||
This release brings small new features and improvements across the codebase.
|
||||
Some highlights:
|
||||
|
||||
* Google Cloud Storage is supported in FilesPipeline and ImagesPipeline.
|
||||
* Crawling with proxy servers becomes more efficient, as connections
|
||||
to proxies can be reused now.
|
||||
* Warnings, exception and logging messages are improved to make debugging
|
||||
easier.
|
||||
* ``scrapy parse`` command now allows to set custom request meta via
|
||||
``--meta`` argument.
|
||||
* Compatibility with Python 3.6, PyPy and PyPy3 is improved;
|
||||
PyPy and PyPy3 are now supported officially, by running tests on CI.
|
||||
* Better default handling of HTTP 308, 522 and 524 status codes.
|
||||
* Documentation is improved, as usual.
|
||||
|
||||
Backwards Incompatible Changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Scrapy 1.5 drops support for Python 3.3.
|
||||
* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
|
||||
**This is technically backwards-incompatible**; override
|
||||
:setting:`USER_AGENT` if you relied on old value.
|
||||
* Logging of settings overridden by ``custom_settings`` is fixed;
|
||||
**this is technically backwards-incompatible** because the logger
|
||||
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're
|
||||
parsing Scrapy logs, please update your log parsers (:issue:`1343`).
|
||||
* LinkExtractor now ignores ``m4v`` extension by default, this is change
|
||||
in behavior.
|
||||
* 522 and 524 status codes are added to ``RETRY_HTTP_CODES`` (:issue:`2851`)
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Support ``<link>`` tags in ``Response.follow`` (:issue:`2785`)
|
||||
- Support for ``ptpython`` REPL (:issue:`2654`)
|
||||
- Google Cloud Storage support for FilesPipeline and ImagesPipeline
|
||||
(:issue:`2923`).
|
||||
- New ``--meta`` option of the "scrapy parse" command allows to pass additional
|
||||
request.meta (:issue:`2883`)
|
||||
- Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`)
|
||||
- Handle HTTP 308 Permanent Redirect (:issue:`2844`)
|
||||
- Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`)
|
||||
- Log versions information at startup (:issue:`2857`)
|
||||
- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0)
|
||||
- Connections to proxy servers are reused (:issue:`2743`)
|
||||
- Add template for a downloader middleware (:issue:`2755`)
|
||||
- Explicit message for NotImplementedError when parse callback not defined
|
||||
(:issue:`2831`)
|
||||
- CrawlerProcess got an option to disable installation of root log handler
|
||||
(:issue:`2921`)
|
||||
- LinkExtractor now ignores ``m4v`` extension by default
|
||||
- Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and
|
||||
:setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`)
|
||||
- Show warning when a URL is put to ``Spider.allowed_domains`` instead of
|
||||
a domain (:issue:`2250`).
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- Fix logging of settings overridden by ``custom_settings``;
|
||||
**this is technically backwards-incompatible** because the logger
|
||||
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please
|
||||
update your log parsers if needed (:issue:`1343`)
|
||||
- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
|
||||
**This is technically backwards-incompatible**; override
|
||||
:setting:`USER_AGENT` if you relied on old value.
|
||||
- Fix PyPy and PyPy3 test failures, support them officially
|
||||
(:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`,
|
||||
:issue:`3048`)
|
||||
- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`)
|
||||
- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`)
|
||||
- Add verification to check if Request callback is callable (:issue:`2766`)
|
||||
- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`)
|
||||
- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning
|
||||
(:issue:`2862`)
|
||||
- Update deprecated test aliases (:issue:`2876`)
|
||||
- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`)
|
||||
|
||||
Docs
|
||||
~~~~
|
||||
|
||||
- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY``
|
||||
setting. (:issue:`2756`)
|
||||
- Update Contributing docs, document new support channels
|
||||
(:issue:`2762`, issue:`3038`)
|
||||
- Include references to Scrapy subreddit in the docs
|
||||
- Fix broken links; use https:// for external links
|
||||
(:issue:`2978`, :issue:`2982`, :issue:`2958`)
|
||||
- Document CloseSpider extension better (:issue:`2759`)
|
||||
- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example
|
||||
(:issue:`2781`)
|
||||
- Spelling mistake and typos
|
||||
(:issue:`2828`, :issue:`2837`, :issue:`2884`, :issue:`2924`)
|
||||
- Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`)
|
||||
- Document ``DontCloseSpider`` exception and clarify ``spider_idle``
|
||||
(:issue:`2791`)
|
||||
- Update "Releases" section in README (:issue:`2764`)
|
||||
- Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`)
|
||||
- Small fix in description of startproject arguments (:issue:`2866`)
|
||||
- Clarify data types in Response.body docs (:issue:`2922`)
|
||||
- Add a note about ``request.meta['depth']`` to DepthMiddleware docs (:issue:`2374`)
|
||||
- Add a note about ``request.meta['dont_merge_cookies']`` to CookiesMiddleware
|
||||
docs (:issue:`2999`)
|
||||
- Up-to-date example of project structure (:issue:`2964`, :issue:`2976`)
|
||||
- A better example of ItemExporters usage (:issue:`2989`)
|
||||
- Document ``from_crawler`` methods for spider and downloader middlewares
|
||||
(:issue:`3019`)
|
||||
|
||||
|
||||
Scrapy 1.4.0 (2017-05-18)
|
||||
-------------------------
|
||||
|
||||
|
|
@ -44,7 +181,7 @@ attributes with ``FormRequest``.
|
|||
|
||||
**Please also note that link extractors do not canonicalize URLs by default
|
||||
anymore.** This was puzzling users every now and then, and it's not what
|
||||
browsers do in fact, so we removed that extra transformation on extractred
|
||||
browsers do in fact, so we removed that extra transformation on extracted
|
||||
links.
|
||||
|
||||
For those of you wanting more control on the ``Referer:`` header that Scrapy
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
Sphinx>=1.3
|
||||
Sphinx>=1.6
|
||||
sphinx_rtd_theme
|
||||
|
|
@ -20,7 +20,7 @@ These are some common properties often found in broad crawls:
|
|||
|
||||
* they crawl many domains (often, unbounded) instead of a specific set of sites
|
||||
|
||||
* they don't necessarily crawl domains to completion, because it would
|
||||
* they don't necessarily crawl domains to completion, because it would be
|
||||
impractical (or impossible) to do so, and instead limit the crawl by time or
|
||||
number of pages crawled
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ structure by default, similar to this::
|
|||
myproject/
|
||||
__init__.py
|
||||
items.py
|
||||
middlewares.py
|
||||
pipelines.py
|
||||
settings.py
|
||||
spiders/
|
||||
|
|
@ -187,7 +188,7 @@ startproject
|
|||
|
||||
Creates a new Scrapy project named ``project_name``, under the ``project_dir``
|
||||
directory.
|
||||
If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``.
|
||||
If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``project_name``.
|
||||
|
||||
Usage example::
|
||||
|
||||
|
|
@ -430,6 +431,9 @@ Supported options:
|
|||
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
||||
response
|
||||
|
||||
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
|
||||
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
|
||||
|
||||
* ``--pipelines``: process items through pipelines
|
||||
|
||||
* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
|
||||
|
|
|
|||
|
|
@ -142,4 +142,4 @@ available in all future runs should they be necessary again::
|
|||
|
||||
For more information, check the :ref:`topics-logging` section.
|
||||
|
||||
.. _base tag: http://www.w3schools.com/tags/tag_base.asp
|
||||
.. _base tag: https://www.w3schools.com/tags/tag_base.asp
|
||||
|
|
|
|||
|
|
@ -51,9 +51,9 @@ just like ``scrapyd-deploy``.
|
|||
|
||||
.. _Scrapyd: https://github.com/scrapy/scrapyd
|
||||
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/
|
||||
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
|
||||
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
|
||||
.. _shub: http://doc.scrapinghub.com/shub.html
|
||||
.. _shub: https://doc.scrapinghub.com/shub.html
|
||||
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||
.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html
|
||||
.. _Scrapinghub: http://scrapinghub.com/
|
||||
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
|
||||
.. _Scrapinghub: https://scrapinghub.com/
|
||||
|
|
|
|||
|
|
@ -157,6 +157,17 @@ more of the following methods:
|
|||
:param spider: the spider for which this request is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this classmethod is called to create a middleware instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the middleware. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for middleware to
|
||||
access them and hook its functionality into Scrapy.
|
||||
|
||||
:param crawler: crawler that uses this middleware
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
.. _topics-downloader-middleware-ref:
|
||||
|
||||
Built-in downloader middleware reference
|
||||
|
|
@ -226,6 +237,15 @@ Default: ``True``
|
|||
Whether to enable the cookies middleware. If disabled, no cookies will be sent
|
||||
to web servers.
|
||||
|
||||
Notice that despite the value of :setting:`COOKIES_ENABLED` setting if
|
||||
``Request.``:reqmeta:`meta['dont_merge_cookies'] <dont_merge_cookies>`
|
||||
evaluates to ``True`` the request cookies will **not** be sent to the
|
||||
web server and received cookies in :class:`~scrapy.http.Response` will
|
||||
**not** be merged with the existing cookies.
|
||||
|
||||
For more detailed information see the ``cookies`` parameter in
|
||||
:class:`~scrapy.http.Request`.
|
||||
|
||||
.. setting:: COOKIES_DEBUG
|
||||
|
||||
COOKIES_DEBUG
|
||||
|
|
@ -862,7 +882,7 @@ precedence over the :setting:`RETRY_TIMES` setting.
|
|||
RETRY_HTTP_CODES
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``[500, 502, 503, 504, 408]``
|
||||
Default: ``[500, 502, 503, 504, 522, 524, 408]``
|
||||
|
||||
Which HTTP response codes to retry. Other errors (DNS lookup issues,
|
||||
connections lost, etc) are always retried.
|
||||
|
|
|
|||
|
|
@ -54,10 +54,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
|
|||
:param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER`
|
||||
setting will be used. If not given, no SMTP authentication will be
|
||||
performed.
|
||||
:type smtphost: str
|
||||
:type smtphost: str or bytes
|
||||
|
||||
:param smtppass: the SMTP pass for authentication.
|
||||
:type smtppass: str
|
||||
:type smtppass: str or bytes
|
||||
|
||||
:param smtpport: the SMTP port to connect to
|
||||
:type smtpport: int
|
||||
|
|
|
|||
|
|
@ -36,38 +36,36 @@ to export
|
|||
3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal
|
||||
the end of the exporting process
|
||||
|
||||
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
|
||||
Exporter to export scraped items to different files, one per spider::
|
||||
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
|
||||
Item Exporters to group scraped items to different files according to the
|
||||
value of one of their fields::
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exporters import XmlItemExporter
|
||||
from scrapy.exporters import XmlItemExporter
|
||||
|
||||
class XmlExportPipeline(object):
|
||||
class PerYearXmlExportPipeline(object):
|
||||
"""Distribute items across multiple XML files according to their 'year' field"""
|
||||
|
||||
def __init__(self):
|
||||
self.files = {}
|
||||
def open_spider(self, spider):
|
||||
self.year_to_exporter = {}
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
pipeline = cls()
|
||||
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
|
||||
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
|
||||
return pipeline
|
||||
def close_spider(self, spider):
|
||||
for exporter in self.year_to_exporter.values():
|
||||
exporter.finish_exporting()
|
||||
exporter.file.close()
|
||||
|
||||
def spider_opened(self, spider):
|
||||
file = open('%s_products.xml' % spider.name, 'w+b')
|
||||
self.files[spider] = file
|
||||
self.exporter = XmlItemExporter(file)
|
||||
self.exporter.start_exporting()
|
||||
def _exporter_for_item(self, item):
|
||||
year = item['year']
|
||||
if year not in self.year_to_exporter:
|
||||
f = open('{}.xml'.format(year), 'wb')
|
||||
exporter = XmlItemExporter(f)
|
||||
exporter.start_exporting()
|
||||
self.year_to_exporter[year] = exporter
|
||||
return self.year_to_exporter[year]
|
||||
|
||||
def spider_closed(self, spider):
|
||||
self.exporter.finish_exporting()
|
||||
file = self.files.pop(spider)
|
||||
file.close()
|
||||
|
||||
def process_item(self, item, spider):
|
||||
self.exporter.export_item(item)
|
||||
return item
|
||||
def process_item(self, item, spider):
|
||||
exporter = self._exporter_for_item(item)
|
||||
exporter.export_item(item)
|
||||
return item
|
||||
|
||||
|
||||
.. _topics-exporters-field-serialization:
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ An integer which specifies a number of items. If the spider scrapes more than
|
|||
that amount and those items are passed by the item pipeline, the
|
||||
spider will be closed with the reason ``closespider_itemcount``.
|
||||
Requests which are currently in the downloader queue (up to
|
||||
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
|
||||
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
|
||||
If zero (or non set), spiders won't be closed by number of passed items.
|
||||
|
||||
.. setting:: CLOSESPIDER_PAGECOUNT
|
||||
|
|
@ -373,4 +373,4 @@ For more info see `Debugging in Python`.
|
|||
This extension only works on POSIX-compliant platforms (ie. not Windows).
|
||||
|
||||
.. _Python debugger: https://docs.python.org/2/library/pdb.html
|
||||
.. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html
|
||||
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ The feeds are stored on `Amazon S3`_.
|
|||
* ``s3://mybucket/path/to/export.csv``
|
||||
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `botocore`_ or `boto`_
|
||||
* Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
|
||||
|
||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||
passed through the following settings:
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ In this example, we'll show how to use `Firebug`_ to scrape data from the
|
|||
Project`_ used in the :ref:`tutorial <intro-tutorial>` but with a different
|
||||
face.
|
||||
|
||||
.. _Firebug: http://getfirebug.com
|
||||
.. _Firebug: https://getfirebug.com/
|
||||
.. _Google Directory: http://directory.google.com/
|
||||
.. _Open Directory Project: http://www.dmoz.org
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one
|
|||
after applying some browser clean up and executing Javascript code. Firefox,
|
||||
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
|
||||
the other hand, does not modify the original page HTML, so you won't be able to
|
||||
extract any data if you use ``<tbody>`` in your XPath expressions.
|
||||
extract any data if you use ``<tbody>`` in your XPath expressions.
|
||||
|
||||
Therefore, you should keep in mind the following things when working with
|
||||
Firefox and XPath:
|
||||
|
|
@ -71,11 +71,11 @@ Firecookie
|
|||
|
||||
`Firecookie`_ makes it easier to view and manage cookies. You can use this
|
||||
extension to create a new cookie, delete existing cookies, see a list of cookies
|
||||
for the current site, manage cookies permissions and a lot more.
|
||||
for the current site, manage cookies permissions and a lot more.
|
||||
|
||||
.. _Firebug: http://getfirebug.com
|
||||
.. _Firebug: https://getfirebug.com/
|
||||
.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
|
||||
.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
|
||||
.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
|
||||
.. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/
|
||||
.. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/
|
||||
.. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/
|
||||
|
|
|
|||
|
|
@ -100,4 +100,4 @@ If you wish to log the requests that couldn't be serialized, you can set the
|
|||
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
||||
It is ``False`` by default.
|
||||
|
||||
.. _pickle: http://docs.python.org/library/pickle.html
|
||||
.. _pickle: https://docs.python.org/library/pickle.html
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are
|
|||
other cases where the memory leaks could come from other (more or less obscure)
|
||||
objects. If this is your case, and you can't find your leaks using ``trackref``,
|
||||
you still have another resource: the `Guppy library`_.
|
||||
If you're using Python3, see :ref:`topics-leaks-muppy`.
|
||||
|
||||
.. _Guppy library: https://pypi.python.org/pypi/guppy
|
||||
|
||||
|
|
@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the
|
|||
|
||||
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
|
||||
|
||||
.. _topics-leaks-muppy:
|
||||
|
||||
Debugging memory leaks with muppy
|
||||
=================================
|
||||
If you're using Python 3, you can use muppy from `Pympler`_.
|
||||
|
||||
.. _Pympler: https://pypi.org/project/Pympler/
|
||||
|
||||
If you use ``pip``, you can install muppy with the following command::
|
||||
|
||||
pip install Pympler
|
||||
|
||||
Here's an example to view all Python objects available in
|
||||
the heap using muppy::
|
||||
|
||||
>>> from pympler import muppy
|
||||
>>> all_objects = muppy.get_objects()
|
||||
>>> len(all_objects)
|
||||
28667
|
||||
>>> from pympler import summary
|
||||
>>> suml = summary.summarize(all_objects)
|
||||
>>> summary.print_(suml)
|
||||
types | # objects | total size
|
||||
==================================== | =========== | ============
|
||||
<class 'str | 9822 | 1.10 MB
|
||||
<class 'dict | 1658 | 856.62 KB
|
||||
<class 'type | 436 | 443.60 KB
|
||||
<class 'code | 2974 | 419.56 KB
|
||||
<class '_io.BufferedWriter | 2 | 256.34 KB
|
||||
<class 'set | 420 | 159.88 KB
|
||||
<class '_io.BufferedReader | 1 | 128.17 KB
|
||||
<class 'wrapper_descriptor | 1130 | 88.28 KB
|
||||
<class 'tuple | 1304 | 86.57 KB
|
||||
<class 'weakref | 1013 | 79.14 KB
|
||||
<class 'builtin_function_or_method | 958 | 67.36 KB
|
||||
<class 'method_descriptor | 865 | 60.82 KB
|
||||
<class 'abc.ABCMeta | 62 | 59.96 KB
|
||||
<class 'list | 446 | 58.52 KB
|
||||
<class 'int | 1425 | 43.20 KB
|
||||
|
||||
For more info about muppy, refer to the `muppy documentation`_.
|
||||
|
||||
.. _muppy documentation: https://pythonhosted.org/Pympler/muppy.html
|
||||
|
||||
.. _topics-leaks-without-leaks:
|
||||
|
||||
Leaks without leaks
|
||||
|
|
|
|||
|
|
@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator.
|
|||
containing the collected values (for that field). The result of the output
|
||||
processors is the value that will be finally assigned to the item.
|
||||
|
||||
If you want to use a plain function as a processor, make sure it receives
|
||||
``self`` as the first argument::
|
||||
|
||||
def lowercase_processor(self, values):
|
||||
for v in values:
|
||||
yield v.lower()
|
||||
|
||||
class MyItemLoader(ItemLoader):
|
||||
name_in = lowercase_processor
|
||||
|
||||
This is because whenever a function is assigned as a class variable, it becomes
|
||||
a method and would be passed the instance as the the first argument when being
|
||||
called. See `this answer on stackoverflow`_ for more details.
|
||||
|
||||
The other thing you need to keep in mind is that the values returned by input
|
||||
processors are collected internally (in lists) and then passed to output
|
||||
processors to populate the fields.
|
||||
|
|
@ -143,6 +157,7 @@ processors to populate the fields.
|
|||
Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
||||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
|
||||
.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
|
||||
|
||||
Declaring Item Loaders
|
||||
======================
|
||||
|
|
@ -518,8 +533,8 @@ a footer of a page that looks something like:
|
|||
Example::
|
||||
|
||||
<footer>
|
||||
<a class="social" href="http://facebook.com/whatever">Like Us</a>
|
||||
<a class="social" href="http://twitter.com/whatever">Follow Us</a>
|
||||
<a class="social" href="https://facebook.com/whatever">Like Us</a>
|
||||
<a class="social" href="https://twitter.com/whatever">Follow Us</a>
|
||||
<a class="email" href="mailto:whatever@example.com">Email Us</a>
|
||||
</footer>
|
||||
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ instance, which can be accessed and used like this::
|
|||
class MySpider(scrapy.Spider):
|
||||
|
||||
name = 'myspider'
|
||||
start_urls = ['http://scrapinghub.com']
|
||||
start_urls = ['https://scrapinghub.com']
|
||||
|
||||
def parse(self, response):
|
||||
self.logger.info('Parse function called on %s', response.url)
|
||||
|
|
@ -118,7 +118,7 @@ Python logger you want. For example::
|
|||
class MySpider(scrapy.Spider):
|
||||
|
||||
name = 'myspider'
|
||||
start_urls = ['http://scrapinghub.com']
|
||||
start_urls = ['https://scrapinghub.com']
|
||||
|
||||
def parse(self, response):
|
||||
logger.info('Parse function called on %s', response.url)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
|
|||
Both pipelines implement these features:
|
||||
|
||||
* Avoid re-downloading media that was downloaded recently
|
||||
* Specifying where to store the media (filesystem directory, Amazon S3 bucket)
|
||||
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
|
||||
Google Cloud Storage bucket)
|
||||
|
||||
The Images Pipeline has a few extra functions for processing images:
|
||||
|
||||
|
|
@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
|
|||
Supported Storage
|
||||
=================
|
||||
|
||||
File system is currently the only officially supported storage, but there is
|
||||
also support for storing files in `Amazon S3`_.
|
||||
File system is currently the only officially supported storage, but there are
|
||||
also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
|
||||
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _Google Cloud Storage: https://cloud.google.com/storage/
|
||||
|
||||
File system storage
|
||||
-------------------
|
||||
|
|
@ -169,7 +171,53 @@ policy::
|
|||
|
||||
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
|
||||
|
||||
.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
|
||||
|
||||
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
|
||||
|
||||
For self-hosting you also might feel the need not to use SSL and not to verify SSL connection::
|
||||
|
||||
AWS_USE_SSL = False # or True (None by default)
|
||||
AWS_VERIFY = False # or True (None by default)
|
||||
|
||||
.. _Minio: https://github.com/minio/minio
|
||||
.. _s3.scality: https://s3.scality.com/
|
||||
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
||||
Google Cloud Storage
|
||||
---------------------
|
||||
|
||||
.. setting:: GCS_PROJECT_ID
|
||||
.. setting:: FILES_STORE_GCS_ACL
|
||||
.. setting:: IMAGES_STORE_GCS_ACL
|
||||
|
||||
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
|
||||
bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
|
||||
|
||||
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
|
||||
|
||||
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
|
||||
|
||||
IMAGES_STORE = 'gs://bucket/images/'
|
||||
GCS_PROJECT_ID = 'project_id'
|
||||
|
||||
For information about authentication, see this `documentation`_.
|
||||
|
||||
.. _documentation: https://cloud.google.com/docs/authentication/production
|
||||
|
||||
You can modify the Access Control List (ACL) policy used for the stored files,
|
||||
which is defined by the :setting:`FILES_STORE_GCS_ACL` and
|
||||
:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to
|
||||
``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object.
|
||||
To make the files publicly available use the ``publicRead``
|
||||
policy::
|
||||
|
||||
IMAGES_STORE_GCS_ACL = 'publicRead'
|
||||
|
||||
For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide.
|
||||
|
||||
.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl
|
||||
|
||||
Usage example
|
||||
=============
|
||||
|
|
|
|||
|
|
@ -248,10 +248,10 @@ If you are still unable to prevent your bot getting banned, consider contacting
|
|||
`commercial support`_.
|
||||
|
||||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: http://scrapy.org/support/
|
||||
.. _ProxyMesh: http://proxymesh.com/
|
||||
.. _commercial support: https://scrapy.org/support/
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
.. _Google cache: http://www.googleguide.com/cached_pages.html
|
||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||
.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
|
||||
.. _Crawlera: http://scrapinghub.com/crawlera
|
||||
.. _scrapoxy: http://scrapoxy.io/
|
||||
.. _Crawlera: https://scrapinghub.com/crawlera
|
||||
.. _scrapoxy: https://scrapoxy.io/
|
||||
|
|
|
|||
|
|
@ -80,6 +80,8 @@ Request objects
|
|||
attributes of the cookie. This is only useful if the cookies are saved
|
||||
for later requests.
|
||||
|
||||
.. reqmeta:: dont_merge_cookies
|
||||
|
||||
When some site returns cookies (in a response) those are stored in the
|
||||
cookies for that domain and will be sent again in future requests. That's
|
||||
the typical behaviour of any regular web browser. However, if, for some
|
||||
|
|
@ -294,7 +296,7 @@ Those are:
|
|||
* :reqmeta:`dont_retry`
|
||||
* :reqmeta:`handle_httpstatus_list`
|
||||
* :reqmeta:`handle_httpstatus_all`
|
||||
* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor)
|
||||
* :reqmeta:`dont_merge_cookies`
|
||||
* :reqmeta:`cookiejar`
|
||||
* :reqmeta:`dont_cache`
|
||||
* :reqmeta:`redirect_urls`
|
||||
|
|
@ -525,11 +527,11 @@ Response objects
|
|||
(for single valued headers) or lists (for multi-valued headers).
|
||||
:type headers: dict
|
||||
|
||||
:param body: the response body. It must be str, not unicode, unless you're
|
||||
using a encoding-aware :ref:`Response subclass
|
||||
<topics-request-response-ref-response-subclasses>`, such as
|
||||
:class:`TextResponse`.
|
||||
:type body: str
|
||||
:param body: the response body. To access the decoded text as str (unicode
|
||||
in Python 2) you can use ``response.text`` from an encoding-aware
|
||||
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
|
||||
such as :class:`TextResponse`.
|
||||
:type body: bytes
|
||||
|
||||
:param flags: is a list containing the initial values for the
|
||||
:attr:`Response.flags` attribute. If given, the list will be shallow
|
||||
|
|
@ -734,7 +736,7 @@ HtmlResponse objects
|
|||
which adds encoding auto-discovering support by looking into the HTML `meta
|
||||
http-equiv`_ attribute. See :attr:`TextResponse.encoding`.
|
||||
|
||||
.. _meta http-equiv: http://www.w3schools.com/TAGS/att_meta_http_equiv.asp
|
||||
.. _meta http-equiv: https://www.w3schools.com/TAGS/att_meta_http_equiv.asp
|
||||
|
||||
XmlResponse objects
|
||||
-------------------
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ documents.
|
|||
For a complete reference of the selectors API see
|
||||
:ref:`Selector reference <topics-selectors-ref>`
|
||||
|
||||
.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
|
||||
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
|
||||
.. _lxml: http://lxml.de/
|
||||
.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
|
||||
.. _cssselect: https://pypi.python.org/pypi/cssselect/
|
||||
|
|
@ -86,7 +86,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which
|
|||
provides interactive testing) and an example page located in the Scrapy
|
||||
documentation server:
|
||||
|
||||
http://doc.scrapy.org/en/latest/_static/selectors-sample1.html
|
||||
https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
|
||||
|
||||
.. _topics-selectors-htmlcode:
|
||||
|
||||
|
|
@ -99,7 +99,7 @@ Here's its HTML code:
|
|||
|
||||
First, let's open the shell::
|
||||
|
||||
scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html
|
||||
scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
|
||||
|
||||
Then, after the shell loads, you'll have the response available as ``response``
|
||||
shell variable, and its attached selector in ``response.selector`` attribute.
|
||||
|
|
|
|||
|
|
@ -180,6 +180,34 @@ such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
|||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
AWS_ENDPOINT_URL
|
||||
----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality.
|
||||
|
||||
.. setting:: AWS_ENDPOINT_URL
|
||||
|
||||
AWS_USE_SSL
|
||||
-----------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Use this option if you want to disable SSL connection for communication with S3 or S3-like storage.
|
||||
By default SSL will be used.
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
AWS_VERIFY
|
||||
----------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur.
|
||||
|
||||
.. setting:: AWS_VERIFY
|
||||
|
||||
BOT_NAME
|
||||
--------
|
||||
|
||||
|
|
@ -307,17 +335,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
|
|||
other priority settings :setting:`REDIRECT_PRIORITY_ADJUST`
|
||||
and :setting:`RETRY_PRIORITY_ADJUST`.
|
||||
|
||||
.. setting:: DEPTH_STATS
|
||||
|
||||
DEPTH_STATS
|
||||
-----------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
|
||||
|
||||
Whether to collect maximum depth stats.
|
||||
|
||||
.. setting:: DEPTH_STATS_VERBOSE
|
||||
|
||||
DEPTH_STATS_VERBOSE
|
||||
|
|
@ -1002,7 +1019,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
|||
|
||||
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
|
||||
|
||||
.. _wget: http://www.gnu.org/software/wget/manual/wget.html
|
||||
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
||||
|
||||
.. setting:: REACTOR_THREADPOOL_MAXSIZE
|
||||
|
||||
|
|
@ -1317,14 +1334,14 @@ Default: ``2083``
|
|||
Scope: ``spidermiddlewares.urllength``
|
||||
|
||||
The maximum URL length to allow for crawled URLs. For more information about
|
||||
the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html
|
||||
the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
|
||||
|
||||
.. setting:: USER_AGENT
|
||||
|
||||
USER_AGENT
|
||||
----------
|
||||
|
||||
Default: ``"Scrapy/VERSION (+http://scrapy.org)"``
|
||||
Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
|
||||
|
||||
The default User-Agent to use when crawling, unless overridden.
|
||||
|
||||
|
|
|
|||
|
|
@ -39,9 +39,9 @@ variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
|
|||
[settings]
|
||||
shell = bpython
|
||||
|
||||
.. _IPython: http://ipython.org/
|
||||
.. _IPython installation guide: http://ipython.org/install.html
|
||||
.. _bpython: http://www.bpython-interpreter.org/
|
||||
.. _IPython: https://ipython.org/
|
||||
.. _IPython installation guide: https://ipython.org/install.html
|
||||
.. _bpython: https://www.bpython-interpreter.org/
|
||||
|
||||
Launch the shell
|
||||
================
|
||||
|
|
@ -142,7 +142,7 @@ Example of shell session
|
|||
========================
|
||||
|
||||
Here's an example of a typical shell session where we start by scraping the
|
||||
http://scrapy.org page, and then proceed to scrape the https://reddit.com
|
||||
https://scrapy.org page, and then proceed to scrape the https://reddit.com
|
||||
page. Finally, we modify the (Reddit) request method to POST and re-fetch it
|
||||
getting an error. We end the session by typing Ctrl-D (in Unix systems) or
|
||||
Ctrl-Z in Windows.
|
||||
|
|
@ -154,7 +154,7 @@ shell works.
|
|||
|
||||
First, we launch the shell::
|
||||
|
||||
scrapy shell 'http://scrapy.org' --nolog
|
||||
scrapy shell 'https://scrapy.org' --nolog
|
||||
|
||||
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
|
||||
list of available objects and useful shortcuts (you'll notice that these lines
|
||||
|
|
@ -164,7 +164,7 @@ all start with the ``[s]`` prefix)::
|
|||
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
|
||||
[s] crawler <scrapy.crawler.Crawler object at 0x7f07395dd690>
|
||||
[s] item {}
|
||||
[s] request <GET http://scrapy.org>
|
||||
[s] request <GET https://scrapy.org>
|
||||
[s] response <200 https://scrapy.org/>
|
||||
[s] settings <scrapy.settings.Settings object at 0x7f07395dd710>
|
||||
[s] spider <DefaultSpider 'default' at 0x7f0735891690>
|
||||
|
|
@ -182,7 +182,7 @@ After that, we can start playing with the objects::
|
|||
>>> response.xpath('//title/text()').extract_first()
|
||||
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
|
||||
|
||||
>>> fetch("http://reddit.com")
|
||||
>>> fetch("https://reddit.com")
|
||||
|
||||
>>> response.xpath('//title/text()').extract()
|
||||
['reddit: the front page of the internet']
|
||||
|
|
|
|||
|
|
@ -135,6 +135,29 @@ item_dropped
|
|||
to be dropped
|
||||
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
|
||||
|
||||
item_error
|
||||
------------
|
||||
|
||||
.. signal:: item_error
|
||||
.. function:: item_error(item, response, spider, failure)
|
||||
|
||||
Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
|
||||
an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
|
||||
:param item: the item dropped from the :ref:`topics-item-pipeline`
|
||||
:type item: dict or :class:`~scrapy.item.Item` object
|
||||
|
||||
:param response: the response being processed when the exception was raised
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
:param failure: the exception raised as a Twisted `Failure`_ object
|
||||
:type failure: `Failure`_ object
|
||||
|
||||
spider_closed
|
||||
-------------
|
||||
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ following methods:
|
|||
method (from other spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
iterable of :class:`~scrapy.http.Response`, dict or
|
||||
iterable of :class:`~scrapy.http.Request`, dict or
|
||||
:class:`~scrapy.item.Item` objects.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this exception,
|
||||
|
|
@ -164,6 +164,17 @@ following methods:
|
|||
:param spider: the spider to whom the start requests belong
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this classmethod is called to create a middleware instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the middleware. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for middleware to
|
||||
access them and hook its functionality into Scrapy.
|
||||
|
||||
:param crawler: crawler that uses this middleware
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
|
||||
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
|
||||
|
||||
|
|
@ -188,16 +199,21 @@ DepthMiddleware
|
|||
|
||||
.. class:: DepthMiddleware
|
||||
|
||||
DepthMiddleware is a scrape middleware used for tracking the depth of each
|
||||
Request inside the site being scraped. It can be used to limit the maximum
|
||||
depth to scrape or things like that.
|
||||
DepthMiddleware is used for tracking the depth of each Request inside the
|
||||
site being scraped. It works by setting `request.meta['depth'] = 0` whenever
|
||||
there is no value previously set (usually just the first Request) and
|
||||
incrementing it by 1 otherwise.
|
||||
|
||||
It can be used to limit the maximum depth to scrape, control Request
|
||||
priority based on their depth, and things like that.
|
||||
|
||||
The :class:`DepthMiddleware` can be configured through the following
|
||||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
|
||||
crawl for any site. If zero, no limit will be imposed.
|
||||
* :setting:`DEPTH_STATS` - Whether to collect depth stats.
|
||||
* :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of
|
||||
requests for each depth.
|
||||
* :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
|
||||
their depth.
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ scrapy.Spider
|
|||
|
||||
A list of URLs where the spider will begin to crawl from, when no
|
||||
particular URLs are specified. So, the first pages downloaded will be those
|
||||
listed here. The subsequent URLs will be generated successively from data
|
||||
listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
|
||||
contained in the start URLs.
|
||||
|
||||
.. attribute:: custom_settings
|
||||
|
|
@ -751,8 +751,8 @@ Combine SitemapSpider with other sources of urls::
|
|||
def parse_other(self, response):
|
||||
pass # ... scrape other here ...
|
||||
|
||||
.. _Sitemaps: http://www.sitemaps.org
|
||||
.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index
|
||||
.. _Sitemaps: https://www.sitemaps.org/index.html
|
||||
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
|
||||
.. _robots.txt: http://www.robotstxt.org/
|
||||
.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
|
||||
.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/
|
||||
|
|
|
|||
|
|
@ -37,5 +37,5 @@ To use the packages:
|
|||
.. warning:: `python-scrapy` is a different package provided by official debian
|
||||
repositories, it's very outdated and it isn't supported by Scrapy team.
|
||||
|
||||
.. _Scrapinghub: http://scrapinghub.com/
|
||||
.. _Scrapinghub: https://scrapinghub.com/
|
||||
.. _GitHub repo: https://github.com/scrapy/scrapy
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ _filename = None
|
|||
_contents = None
|
||||
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Run tests, generate coverage report and open it on a browser
|
||||
#
|
||||
# Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage
|
||||
# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage
|
||||
|
||||
coverage run --branch $(which trial) --reporter=text tests
|
||||
coverage html -i
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class QPSSpider(Spider):
|
|||
|
||||
slots = int(self.slots)
|
||||
if slots > 1:
|
||||
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)]
|
||||
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
|
||||
else:
|
||||
urls = [url]
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ Twisted>=13.1.0
|
|||
lxml
|
||||
pyOpenSSL
|
||||
cssselect>=0.9
|
||||
w3lib>=1.17.0
|
||||
queuelib
|
||||
w3lib>=1.17.0
|
||||
six>=1.5.2
|
||||
PyDispatcher>=2.0.5
|
||||
parsel>=1.4
|
||||
service_identity
|
||||
parsel>=1.1
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
Twisted >= 15.5.0
|
||||
Twisted>=17.9.0
|
||||
lxml>=3.2.4
|
||||
pyOpenSSL>=0.13.1
|
||||
cssselect>=0.9
|
||||
queuelib>=1.1.1
|
||||
w3lib>=1.17.0
|
||||
six>=1.5.2
|
||||
PyDispatcher>=2.0.5
|
||||
parsel>=1.4
|
||||
service_identity
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
1.4.0
|
||||
1.5.0
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@ from six.moves import copyreg
|
|||
if sys.version_info[0] == 2:
|
||||
from urlparse import urlparse
|
||||
|
||||
# workaround for http://bugs.python.org/issue7904 - Python < 2.7
|
||||
# workaround for https://bugs.python.org/issue7904 - Python < 2.7
|
||||
if urlparse('s3://bucket/key').netloc != 'bucket':
|
||||
from urlparse import uses_netloc
|
||||
uses_netloc.append('s3')
|
||||
|
||||
# workaround for http://bugs.python.org/issue9374 - Python < 2.7.4
|
||||
# workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
|
||||
if urlparse('s3://bucket/key?key=value').query != 'key=value':
|
||||
from urlparse import uses_query
|
||||
uses_query.append('s3')
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.command` is deprecated, "
|
||||
"use `scrapy.commands` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.commands import *
|
||||
|
|
@ -56,3 +56,6 @@ class Command(ScrapyCommand):
|
|||
|
||||
self.crawler_process.crawl(spname, **opts.spargs)
|
||||
self.crawler_process.start()
|
||||
|
||||
if self.crawler_process.bootstrap_failed:
|
||||
self.exitcode = 1
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import sys, os
|
||||
import sys
|
||||
import os
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from __future__ import print_function
|
||||
import json
|
||||
import logging
|
||||
|
||||
from w3lib.url import is_url
|
||||
|
|
@ -48,6 +49,8 @@ class Command(ScrapyCommand):
|
|||
help="use CrawlSpider rules to discover the callback")
|
||||
parser.add_option("-c", "--callback", dest="callback",
|
||||
help="use this callback for parsing, instead looking for a callback")
|
||||
parser.add_option("-m", "--meta", dest="meta",
|
||||
help="inject extra meta into the Request, it must be a valid raw json string")
|
||||
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
|
||||
help="maximum depth for parsing requests [default: %default]")
|
||||
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
|
||||
|
|
@ -204,6 +207,10 @@ class Command(ScrapyCommand):
|
|||
req.callback = callback
|
||||
return requests
|
||||
|
||||
#update request meta if any extra meta was passed through the --meta/-m opts.
|
||||
if opts.meta:
|
||||
request.meta.update(opts.meta)
|
||||
|
||||
request.meta['_depth'] = 1
|
||||
request.meta['_callback'] = request.callback
|
||||
request.callback = callback
|
||||
|
|
@ -211,11 +218,27 @@ class Command(ScrapyCommand):
|
|||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
||||
self.process_spider_arguments(opts)
|
||||
self.process_request_meta(opts)
|
||||
|
||||
def process_spider_arguments(self, opts):
|
||||
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
|
||||
def process_request_meta(self, opts):
|
||||
|
||||
if opts.meta:
|
||||
try:
|
||||
opts.meta = json.loads(opts.meta)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
|
||||
"Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
|
||||
|
||||
|
||||
def run(self, args, opts):
|
||||
# parse arguments
|
||||
if not len(args) == 1 or not is_url(args[0]):
|
||||
|
|
|
|||
|
|
@ -87,3 +87,6 @@ class Command(ScrapyCommand):
|
|||
|
||||
self.crawler_process.crawl(spidercls, **opts.spargs)
|
||||
self.crawler_process.start()
|
||||
|
||||
if self.crawler_process.bootstrap_failed:
|
||||
self.exitcode = 1
|
||||
|
|
|
|||
|
|
@ -1,12 +1,8 @@
|
|||
from __future__ import print_function
|
||||
import sys
|
||||
import platform
|
||||
|
||||
import twisted
|
||||
import OpenSSL
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.utils.versions import scrapy_components_versions
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
|
@ -27,38 +23,11 @@ class Command(ScrapyCommand):
|
|||
|
||||
def run(self, args, opts):
|
||||
if opts.verbose:
|
||||
import cssselect
|
||||
import parsel
|
||||
import lxml.etree
|
||||
import w3lib
|
||||
|
||||
lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
|
||||
libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
|
||||
|
||||
try:
|
||||
w3lib_version = w3lib.__version__
|
||||
except AttributeError:
|
||||
w3lib_version = "<1.14.3"
|
||||
|
||||
print("Scrapy : %s" % scrapy.__version__)
|
||||
print("lxml : %s" % lxml_version)
|
||||
print("libxml2 : %s" % libxml2_version)
|
||||
print("cssselect : %s" % cssselect.__version__)
|
||||
print("parsel : %s" % parsel.__version__)
|
||||
print("w3lib : %s" % w3lib_version)
|
||||
print("Twisted : %s" % twisted.version.short())
|
||||
print("Python : %s" % sys.version.replace("\n", "- "))
|
||||
print("pyOpenSSL : %s" % self._get_openssl_version())
|
||||
print("Platform : %s" % platform.platform())
|
||||
versions = scrapy_components_versions()
|
||||
width = max(len(n) for (n, _) in versions)
|
||||
patt = "%-{}s : %s".format(width)
|
||||
for name, version in versions:
|
||||
print(patt % (name, version))
|
||||
else:
|
||||
print("Scrapy %s" % scrapy.__version__)
|
||||
|
||||
def _get_openssl_version(self):
|
||||
try:
|
||||
openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
|
||||
.decode('ascii', errors='replace')
|
||||
# pyOpenSSL 0.12 does not expose openssl version
|
||||
except AttributeError:
|
||||
openssl = 'Unknown OpenSSL version'
|
||||
|
||||
return '{} ({})'.format(OpenSSL.version.__version__, openssl)
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.closespider` is deprecated, "
|
||||
"use `scrapy.extensions.closespider` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.closespider import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.corestats` is deprecated, "
|
||||
"use `scrapy.extensions.corestats` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.corestats import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.debug` is deprecated, "
|
||||
"use `scrapy.extensions.debug` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.debug import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.ajaxcrawl` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.ajaxcrawl import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.chunked` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.chunked import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.cookies` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.cookies import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.decompression` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.decompression import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.defaultheaders` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.defaultheaders import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.downloadtimeout` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.downloadtimeout import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.httpauth` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.httpauth import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.httpcache` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.httpcache import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.httpcompression` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.httpcompression import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.httpproxy` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.httpproxy import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.redirect` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.redirect import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.retry` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.retry import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.robotstxt` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.robotstxt import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.stats` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.stats import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, "
|
||||
"use `scrapy.downloadermiddlewares.useragent` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.downloadermiddlewares.useragent import *
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.exporter` is deprecated, "
|
||||
"use `scrapy.exporters` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.exporters import *
|
||||
from scrapy.exporters import PythonItemExporter
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, "
|
||||
"use `scrapy.extensions.feedexport` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.feedexport import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, "
|
||||
"use `scrapy.extensions.httpcache` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.httpcache import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, "
|
||||
"use `scrapy.linkextractors` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.linkextractors import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, "
|
||||
"use `scrapy.linkextractors.htmlparser` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.linkextractors.htmlparser import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, "
|
||||
"use `scrapy.linkextractors.lxmlhtml` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.linkextractors.lxmlhtml import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, "
|
||||
"use `scrapy.linkextractors.regex` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.linkextractors.regex import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, "
|
||||
"use `scrapy.linkextractors.sgml` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.linkextractors.sgml import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.loader` is deprecated, "
|
||||
"use `scrapy.loader` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.loader import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, "
|
||||
"use `scrapy.loader.common` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.loader.common import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, "
|
||||
"use `scrapy.loader.processors` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.loader.processors import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.logstats` is deprecated, "
|
||||
"use `scrapy.extensions.logstats` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.logstats import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, "
|
||||
"use `scrapy.extensions.memdebug` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.memdebug import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.memusage` is deprecated, "
|
||||
"use `scrapy.extensions.memusage` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.extensions.memusage import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, "
|
||||
"use `scrapy.pipelines` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.pipelines import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, "
|
||||
"use `scrapy.pipelines.files` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.pipelines.files import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, "
|
||||
"use `scrapy.pipelines.images` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.pipelines.images import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, "
|
||||
"use `scrapy.pipelines.media` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.pipelines.media import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, "
|
||||
"use `scrapy.spidermiddlewares.depth` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.spidermiddlewares.depth import *
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import warnings
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, "
|
||||
"use `scrapy.spidermiddlewares.httperror` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
from scrapy.spidermiddlewares.httperror import *
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue