diff --git a/.travis.yml b/.travis.yml index ac79a0425..6f3ab511f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,24 @@ language: python -python: 3.5 sudo: false branches: only: - master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -env: - - TOXENV=py27 - - TOXENV=jessie - - TOXENV=py33 - - TOXENV=py35 - - TOXENV=docs +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: 2.7 + env: TOXENV=jessie + - python: 3.3 + env: TOXENV=py33 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 + - python: 3.6 + env: TOXENV=docs install: - pip install -U tox twine wheel codecov script: tox diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 95b4a7e3c..162602248 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,24 +1,41 @@ -# Contributor Code of Conduct +# Contributor Covenant Code of Conduct -As contributors and maintainers of this project, and in the interest of -fostering an open and welcoming community, we pledge to respect all people who -contribute through reporting issues, posting feature requests, updating -documentation, submitting pull requests or patches, and other activities. +## Our Pledge -We are committed to making participation in this project a harassment-free -experience for everyone, regardless of level of experience, gender, gender -identity and expression, sexual orientation, disability, personal appearance, -body size, race, ethnicity, age, religion, or nationality. +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery -* Personal attacks -* Trolling or insulting/derogatory comments +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment -* Publishing other's private information, such as physical or electronic - addresses, without explicit permission -* Other unethical or unprofessional conduct +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions @@ -26,25 +43,32 @@ that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. -By adopting this Code of Conduct, project maintainers commit themselves to -fairly and consistently applying these principles to every aspect of managing -this project. Project maintainers who do not follow or enforce the Code of -Conduct may be permanently removed from the project team. +## Scope This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting a project maintainer at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@scrapinghub.com. All complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. Maintainers are -obligated to maintain confidentiality with regard to the reporter of an -incident. +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 1.3.0, available at -[http://contributor-covenant.org/version/1/3/0/][version] +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/3/0/ +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/README.rst b/README.rst index b72ebf53d..38dda62e5 100644 --- a/README.rst +++ b/README.rst @@ -73,14 +73,17 @@ See http://scrapy.org/community/ Contributing ============ +See http://doc.scrapy.org/en/master/contributing.html + +Code of Conduct +--------------- + Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@scrapinghub.com. -See http://doc.scrapy.org/en/master/contributing.html - Companies using Scrapy ====================== diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ea0cb0ea7..4ac39ad2d 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -12,7 +12,7 @@ Overview The following diagram shows an overview of the Scrapy architecture with its components and an outline of the data flow that takes place inside the system -(shown by the green arrows). A brief description of the components is included +(shown by the red arrows). A brief description of the components is included below with links for more detailed information about them. The data flow is also described below. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 664a7239f..a1bd1e146 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -358,7 +358,7 @@ fields with form data from :class:`Response` objects. The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) Returns a new :class:`FormRequest` object with its form field values pre-populated with those found in the HTML ``
`` element contained @@ -383,6 +383,9 @@ fields with form data from :class:`Response` objects. :param formname: if given, the form with name attribute set to this value will be used. :type formname: string + :param formid: if given, the form with id attribute set to this value will be used. + :type formid: string + :param formxpath: if given, the first form that matches the xpath will be used. :type formxpath: string @@ -421,6 +424,9 @@ fields with form data from :class:`Response` objects. .. versionadded:: 1.1.0 The ``formcss`` parameter. + .. versionadded:: 1.1.0 + The ``formid`` parameter. + Request usage examples ---------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0179e9284..c123c2635 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -297,6 +297,37 @@ Spiders can access arguments in their `__init__` methods:: self.start_urls = ['http://www.example.com/categories/%s' % category] # ... +The default `__init__` method will take any spider arguments +and copy them to the spider as attributes. +The above example can also be written as follows:: + + import scrapy + + class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + +Keep in mind that spider arguments are only strings. +The spider will not do any parsing on its own. +If you were to set the `start_urls` attribute from the command line, +you would have to parse it on your own into a list +using something like +`ast.literal_eval `_ +or `json.loads `_ +and then set it as an attribute. +Otherwise, you would cause iteration over a `start_urls` string +(a very common python pitfall) +resulting in each character being seen as a separate url. + +A valid use case is to set the http auth credentials +used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` +or the user agent +used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: + + scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot + Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 54aa359fb..ecd7f90d3 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -319,14 +319,13 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ("Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize})." - ).format(url=request.url, size=expected_size, maxsize=maxsize) + error_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} - logger.error(error_message) + logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_message) + raise defer.CancelledError(error_msg % error_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " diff --git a/setup.py b/setup.py index f673b1dc4..388cf0dec 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup( 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', diff --git a/tox.ini b/tox.ini index 812302b4c..bdc14a128 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,10 @@ deps = {[testenv:py33]deps} basepython = python3.5 deps = {[testenv:py33]deps} +[testenv:py36] +basepython = python3.6 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps =