From 6dec4a3ccb455511decc4c1414b9042c57a205ea Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 20 Dec 2016 20:02:31 -0400 Subject: [PATCH 01/13] ENH Pass arguments to logger rather than formatted message. This not only use the standard form but helps error aggregation libraries (i.e.: Sentry) to avoid duplicating the message. --- scrapy/core/downloader/handlers/http11.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 54aa359fb..ecd7f90d3 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -319,14 +319,13 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ("Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize})." - ).format(url=request.url, size=expected_size, maxsize=maxsize) + error_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} - logger.error(error_message) + logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_message) + raise defer.CancelledError(error_msg % error_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " From 9922ec15d7a79ac109bf02ccd9ac154bca5d5167 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 27 Dec 2016 09:52:35 -0200 Subject: [PATCH 02/13] mention contributing document before CoC in the README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b72ebf53d..38ad43eba 100644 --- a/README.rst +++ b/README.rst @@ -73,14 +73,14 @@ See http://scrapy.org/community/ Contributing ============ +See http://doc.scrapy.org/en/master/contributing.html + Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@scrapinghub.com. -See http://doc.scrapy.org/en/master/contributing.html - Companies using Scrapy ====================== From dabcb17d72e5bf0ab3a0d7cfe50f70aa2529cd09 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 27 Dec 2016 11:32:15 -0200 Subject: [PATCH 03/13] update code of conduct http://contributor-covenant.org/version/1/4 --- CODE_OF_CONDUCT.md | 80 ++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 95b4a7e3c..162602248 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,24 +1,41 @@ -# Contributor Code of Conduct +# Contributor Covenant Code of Conduct -As contributors and maintainers of this project, and in the interest of -fostering an open and welcoming community, we pledge to respect all people who -contribute through reporting issues, posting feature requests, updating -documentation, submitting pull requests or patches, and other activities. +## Our Pledge -We are committed to making participation in this project a harassment-free -experience for everyone, regardless of level of experience, gender, gender -identity and expression, sexual orientation, disability, personal appearance, -body size, race, ethnicity, age, religion, or nationality. +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery -* Personal attacks -* Trolling or insulting/derogatory comments +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment -* Publishing other's private information, such as physical or electronic - addresses, without explicit permission -* Other unethical or unprofessional conduct +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions @@ -26,25 +43,32 @@ that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. -By adopting this Code of Conduct, project maintainers commit themselves to -fairly and consistently applying these principles to every aspect of managing -this project. Project maintainers who do not follow or enforce the Code of -Conduct may be permanently removed from the project team. +## Scope This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting a project maintainer at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@scrapinghub.com. All complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. Maintainers are -obligated to maintain confidentiality with regard to the reporter of an -incident. +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 1.3.0, available at -[http://contributor-covenant.org/version/1/3/0/][version] +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/3/0/ +[version]: http://contributor-covenant.org/version/1/4/ From b7dd089bf9d041218012ce1da7bce5045e76d37a Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 27 Dec 2016 11:33:23 -0200 Subject: [PATCH 04/13] show CoC in its own section --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 38ad43eba..38dda62e5 100644 --- a/README.rst +++ b/README.rst @@ -75,6 +75,9 @@ Contributing See http://doc.scrapy.org/en/master/contributing.html +Code of Conduct +--------------- + Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). From a21473147160e4b176fedb32c92e1b68b4af04f6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 15:38:35 +0100 Subject: [PATCH 05/13] Add Python 3.6 tox env + Travis CI build for it --- .travis.yml | 1 + tox.ini | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 506f3779b..b0ac3afde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 + - TOXENV=py36 - TOXENV=docs install: - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index 812302b4c..bdc14a128 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,10 @@ deps = {[testenv:py33]deps} basepython = python3.5 deps = {[testenv:py33]deps} +[testenv:py36] +basepython = python3.6 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps = From 40851a3d4cde6b44cc92e03b7272b3acd33ed7f6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 15:44:32 +0100 Subject: [PATCH 06/13] Use Python 3.6-dev on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b0ac3afde..5c9eb8cc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 3.5 +python: 3.6-dev sudo: false branches: only: From 53769245f553355ad1907ad0b47a05093ec65cbe Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 16:02:14 +0100 Subject: [PATCH 07/13] Use python 3.6 directly --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5c9eb8cc7..4c4adb948 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 3.6-dev +python: 3.6 sudo: false branches: only: From 6b838b02966902311ab869bb6a35e023265ed274 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 16:10:14 +0100 Subject: [PATCH 08/13] Use matrix build config --- .travis.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c4adb948..52fbf02ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,24 @@ language: python -python: 3.6 sudo: false branches: only: - master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ -env: - - TOXENV=py27 - - TOXENV=jessie - - TOXENV=py33 - - TOXENV=py35 - - TOXENV=py36 - - TOXENV=docs +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: 2.7 + env: TOXENV=jessie + - python: 3.3 + env: TOXENV=py33 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 + - python: 3.6 + env: TOXENV=docs install: - pip install -U tox twine wheel codecov script: tox From b3406677b980656751af9a12ddf2d33ad884fbc1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 9 Jan 2017 14:40:02 +0100 Subject: [PATCH 09/13] Update classifiers in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f673b1dc4..388cf0dec 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup( 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', From 900b6710d3c9d70c740707d72c611c50d909616b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Thu, 5 Jan 2017 12:53:00 +0200 Subject: [PATCH 10/13] Document copying of spider arguments to attributes --- docs/topics/spiders.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0179e9284..29106e87a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -297,6 +297,37 @@ Spiders can access arguments in their `__init__` methods:: self.start_urls = ['http://www.example.com/categories/%s' % category] # ... +The default `__init__` method will take any spider arguments +and copy them to the spider as attributes. +The above example can also be written as follows:: + + import scrapy + + class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + +Keep in mind that spider arguments are only strings. +The spider will not do any parsing on its own. +If you were to set the `start_urls` attribute from the command line, +you would have to parse it on you own into a list +using something like +`ast.literal_eval `_ +or `json.loads `_ +and then set it as an attribute. +Otherwise, you would cause iteration over a `start_urls` string +(a very common python pitfall) +resulting in each character being seen as a separate url. + +A valid use case is to set the http auth credentials +used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` +or the user agent +used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: + + scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot + Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. From bf2277a028aee0feaecd90a3a64c53bba4e5fbdf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 10 Jan 2017 10:27:38 +0100 Subject: [PATCH 11/13] Update spiders.rst --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 29106e87a..c123c2635 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -312,7 +312,7 @@ The above example can also be written as follows:: Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. If you were to set the `start_urls` attribute from the command line, -you would have to parse it on you own into a list +you would have to parse it on your own into a list using something like `ast.literal_eval `_ or `json.loads `_ From 5586fc7e3876d5a5e8d1797c73cb9b743c8097d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paulius=20Aleksi=C5=ABnas?= Date: Tue, 10 Jan 2017 11:12:42 +0200 Subject: [PATCH 12/13] Update architecture.rst In the data flow image arrows are red. --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ea0cb0ea7..4ac39ad2d 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -12,7 +12,7 @@ Overview The following diagram shows an overview of the Scrapy architecture with its components and an outline of the data flow that takes place inside the system -(shown by the green arrows). A brief description of the components is included +(shown by the red arrows). A brief description of the components is included below with links for more detailed information about them. The data flow is also described below. From df1a42419f8bce48b605087937320af1ec968116 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Sat, 14 Jan 2017 20:45:20 -0500 Subject: [PATCH 13/13] adding formid to FormRequest documentation --- docs/topics/request-response.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 664a7239f..a1bd1e146 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -358,7 +358,7 @@ fields with form data from :class:`Response` objects. The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) Returns a new :class:`FormRequest` object with its form field values pre-populated with those found in the HTML ``
`` element contained @@ -383,6 +383,9 @@ fields with form data from :class:`Response` objects. :param formname: if given, the form with name attribute set to this value will be used. :type formname: string + :param formid: if given, the form with id attribute set to this value will be used. + :type formid: string + :param formxpath: if given, the first form that matches the xpath will be used. :type formxpath: string @@ -421,6 +424,9 @@ fields with form data from :class:`Response` objects. .. versionadded:: 1.1.0 The ``formcss`` parameter. + .. versionadded:: 1.1.0 + The ``formid`` parameter. + Request usage examples ----------------------