RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use pathlib

This commit is contained in:
Matthew Donoughe 2022-10-17 17:40:10 -04:00
parent 1c7f3ebd75
commit 40d9ca3bdd
No known key found for this signature in database
GPG Key ID: 838812402CA8C19D
60 changed files with 595 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conftest.py
View File

@ -21,7 +21,7 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"), *_py_files("tests/CrawlerRunner"),
] ]
with open('tests/ignores.txt') as reader: with Path('tests/ignores.txt').open() as reader:
for line in reader: for line in reader:
file_path = line.strip() file_path = line.strip()
if file_path and file_path[0] != '#': if file_path and file_path[0] != '#':

4
docs/Makefile
View File

@ -86,8 +86,8 @@ coverage: BUILDER = coverage
coverage: build coverage: build
htmlview: html htmlview: html
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ $(PYTHON) -c "import webbrowser; from pathlib import Path; \
os.path.realpath('build/html/index.html'))" webbrowser.open('file://' + Path('build/html/index.html').resolve())"
clean: clean:
-rm -rf build/* -rm -rf build/*

9
docs/conf.py
View File

@ -11,13 +11,12 @@
import sys import sys
from datetime import datetime from datetime import datetime
from os import path from pathlib import Path
# If your extensions are in another directory, add it here. If the directory # If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it # is relative to the documentation root, use Path.absolute to make it absolute.
# absolute, like shown here. sys.path.append(str(Path(__file__).parent / "_ext"))
sys.path.append(path.join(path.dirname(__file__), "_ext")) sys.path.insert(0, str(Path(__file__).parent.parent))
sys.path.insert(0, path.dirname(path.dirname(__file__)))
# General configuration # General configuration

9
docs/conftest.py
View File

@ -1,5 +1,5 @@
import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from pathlib import Path
from scrapy.http.response.html import HtmlResponse from scrapy.http.response.html import HtmlResponse
from sybil import Sybil from sybil import Sybil
@ -12,10 +12,9 @@ from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip from sybil.parsers.skip import skip
def load_response(url, filename): def load_response(url: str, filename: str) -> HtmlResponse:
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename) input_path = Path(__file__).parent / '_tests' / filename
with open(input_path, 'rb') as input_file: return HtmlResponse(url, body=input_path.read_bytes())
return HtmlResponse(url, body=input_file.read())
def setup(namespace): def setup(namespace):

10
docs/intro/tutorial.rst
View File

@ -85,6 +85,8 @@ page content to extract data.
This is the code for our first Spider. Save it in a file named This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
from pathlib import Path
import scrapy import scrapy
@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response): def parse(self, response):
page = response.url.split("/")[-2] page = response.url.split("/")[-2]
filename = f'quotes-{page}.html' filename = f'quotes-{page}.html'
with open(filename, 'wb') as f: Path(filename).write_bytes(response.body)
f.write(response.body)
self.log(f'Saved file {filename}') self.log(f'Saved file {filename}')
@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider:: for your spider::
from pathlib import Path
import scrapy import scrapy
@ -191,8 +194,7 @@ for your spider::
def parse(self, response): def parse(self, response):
page = response.url.split("/")[-2] page = response.url.split("/")[-2]
filename = f'quotes-{page}.html' filename = f'quotes-{page}.html'
with open(filename, 'wb') as f: Path(filename).write_bytes(response.body)
f.write(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy of the requests for those URLs, even though we haven't explicitly told Scrapy

4
docs/topics/item-pipeline.rst
View File

@ -186,6 +186,7 @@ item.
:: ::
import hashlib import hashlib
from pathlib import Path
from urllib.parse import quote from urllib.parse import quote
import scrapy import scrapy
@ -214,8 +215,7 @@ item.
url = adapter["url"] url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest() url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = f"{url_hash}.png" filename = f"{url_hash}.png"
with open(filename, "wb") as f: Path(filename).write_bytes(response.body)
f.write(response.body)
# Store filename in item. # Store filename in item.
adapter["screenshot_filename"] = filename adapter["screenshot_filename"] = filename

9
docs/topics/media-pipeline.rst
View File

@ -156,7 +156,6 @@ By overriding ``file_path`` like this:
.. code-block:: python .. code-block:: python
import hashlib import hashlib
from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None): def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5) image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline:
approach to download all files into the ``files`` folder with their approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``):: original filenames (e.g. ``files/foo.png``)::
import os from pathlib import PurePosixPath
from urllib.parse import urlparse from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline from scrapy.pipelines.files import FilesPipeline
@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline): class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None): def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path) return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item Similarly, you can use the ``item`` to determine the file path based on some item
property. property.
@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline:
approach to download all files into the ``files`` folder with their approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``):: original filenames (e.g. ``files/foo.png``)::
import os from pathlib import PurePosixPath
from urllib.parse import urlparse from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.images import ImagesPipeline
@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline): class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None): def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path) return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item Similarly, you can use the ``item`` to determine the file path based on some item
property. property.

9
docs/utils/linkfix.py
View File

@ -13,6 +13,7 @@ Author: dufferzafar
""" """
import re import re
from pathlib import Path
def main(): def main():
@ -27,7 +28,7 @@ def main():
# Read lines from the linkcheck output file # Read lines from the linkcheck output file
try: try:
with open("build/linkcheck/output.txt") as out: with Path("build/linkcheck/output.txt").open() as out:
output_lines = out.readlines() output_lines = out.readlines()
except IOError: except IOError:
print("linkcheck output not found; please run linkcheck first.") print("linkcheck output not found; please run linkcheck first.")
@ -51,14 +52,12 @@ def main():
# Update the previous file # Update the previous file
if _filename: if _filename:
with open(_filename, "w") as _file: Path(_filename).write_text(_contents)
_file.write(_contents)
_filename = newfilename _filename = newfilename
# Read the new file to memory # Read the new file to memory
with open(_filename) as _file: _contents = Path(_filename).read_text()
_contents = _file.read()
_contents = _contents.replace(match.group(3), match.group(4)) _contents = _contents.replace(match.group(3), match.group(4))
else: else:

4
scrapy/commands/__init__.py
View File

@ -3,6 +3,7 @@ Base class for Scrapy commands
""" """
import os import os
import argparse import argparse
from pathlib import Path
from typing import Any, Dict from typing import Any, Dict
from twisted.python import failure from twisted.python import failure
@ -93,8 +94,7 @@ class ScrapyCommand:
self.settings.set('LOG_ENABLED', False, priority='cmdline') self.settings.set('LOG_ENABLED', False, priority='cmdline')
if opts.pidfile: if opts.pidfile:
with open(opts.pidfile, "w") as f: Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep)
f.write(str(os.getpid()) + os.linesep)
if opts.pdb: if opts.pdb:
failure.startDebugMode() failure.startDebugMode()

48
scrapy/commands/genspider.py
View File

@ -2,8 +2,9 @@ import os
import shutil import shutil
import string import string
from pathlib import Path
from importlib import import_module from importlib import import_module
from os.path import join, dirname, abspath, exists, splitext from typing import Optional
from urllib.parse import urlparse from urllib.parse import urlparse
import scrapy import scrapy
@ -62,8 +63,7 @@ class Command(ScrapyCommand):
if opts.dump: if opts.dump:
template_file = self._find_template(opts.dump) template_file = self._find_template(opts.dump)
if template_file: if template_file:
with open(template_file, "r") as f: print(template_file.read_text())
print(f.read())
return return
if len(args) != 2: if len(args) != 2:
raise UsageError() raise UsageError()
@ -98,11 +98,11 @@ class Command(ScrapyCommand):
} }
if self.settings.get('NEWSPIDER_MODULE'): if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = abspath(dirname(spiders_module.__file__)) spiders_dir = Path(spiders_module.__file__).parent.resolve()
else: else:
spiders_module = None spiders_module = None
spiders_dir = "." spiders_dir = Path(".")
spider_file = f"{join(spiders_dir, module)}.py" spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file) shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars) render_templatefile(spider_file, **tvars)
print(f"Created spider {name!r} using template {template_name!r} ", print(f"Created spider {name!r} using template {template_name!r} ",
@ -110,24 +110,25 @@ class Command(ScrapyCommand):
if spiders_module: if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}") print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template): def _find_template(self, template: str) -> Optional[Path]:
template_file = join(self.templates_dir, f'{template}.tmpl') template_file = Path(self.templates_dir, f'{template}.tmpl')
if exists(template_file): if template_file.exists():
return template_file return template_file
print(f"Unable to find template: {template}\n") print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.') print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self): def _list_templates(self):
print("Available templates:") print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)): for file in sorted(Path(self.templates_dir).iterdir()):
if filename.endswith('.tmpl'): if file.suffix == '.tmpl':
print(f" {splitext(filename)[0]}") print(f" {file.stem}")
def _spider_exists(self, name): def _spider_exists(self, name: str) -> bool:
if not self.settings.get('NEWSPIDER_MODULE'): if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists # if run as a standalone command and file with same filename already exists
if exists(name + ".py"): path = Path(name + ".py")
print(f"{abspath(name + '.py')} already exists") if path.exists():
print(f"{path.resolve()} already exists")
return True return True
return False return False
@ -143,17 +144,18 @@ class Command(ScrapyCommand):
# a file with the same name exists in the target directory # a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__) spiders_dir = Path(spiders_module.__file__).parent
spiders_dir_abs = abspath(spiders_dir) spiders_dir_abs = spiders_dir.resolve()
if exists(join(spiders_dir_abs, name + ".py")): path = spiders_dir_abs / (name + ".py")
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists") if path.exists():
print(f"{path} already exists")
return True return True
return False return False
@property @property
def templates_dir(self): def templates_dir(self) -> str:
return join( return str(Path(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'spiders' 'spiders'
) ))

21
scrapy/commands/runspider.py
View File

@ -1,22 +1,23 @@
import sys import sys
import os from os import PathLike
from pathlib import Path
from importlib import import_module from importlib import import_module
from types import ModuleType
from scrapy.utils.spider import iter_spider_classes from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand from scrapy.commands import BaseRunSpiderCommand
def _import_file(filepath): def _import_file(filepath: str | PathLike[str]) -> ModuleType:
abspath = os.path.abspath(filepath) abspath = Path(filepath).resolve()
dirname, file = os.path.split(abspath) dirname = str(abspath.parent)
fname, fext = os.path.splitext(file) if abspath.suffix not in ('.py', '.pyw'):
if fext not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}") raise ValueError(f"Not a Python source file: {abspath}")
if dirname: if dirname:
sys.path = [dirname] + sys.path sys.path = [dirname] + sys.path
try: try:
module = import_module(fname) module = import_module(abspath.stem)
finally: finally:
if dirname: if dirname:
sys.path.pop(0) sys.path.pop(0)
@ -40,13 +41,13 @@ class Command(BaseRunSpiderCommand):
def run(self, args, opts): def run(self, args, opts):
if len(args) != 1: if len(args) != 1:
raise UsageError() raise UsageError()
filename = args[0] filename = Path(args[0])
if not os.path.exists(filename): if not filename.exists():
raise UsageError(f"File not found: {filename}\n") raise UsageError(f"File not found: {filename}\n")
try: try:
module = _import_file(filename) module = _import_file(filename)
except (ImportError, ValueError) as e: except (ImportError, ValueError) as e:
raise UsageError(f"Unable to load {filename!r}: {e}\n") raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
spclasses = list(iter_spider_classes(module)) spclasses = list(iter_spider_classes(module))
if not spclasses: if not spclasses:
raise UsageError(f"No spider found in file: {filename}\n") raise UsageError(f"No spider found in file: {filename}\n")

44
scrapy/commands/startproject.py
View File

@ -2,7 +2,7 @@ import re
import os import os
import string import string
from importlib.util import find_spec from importlib.util import find_spec
from os.path import join, exists, abspath from pathlib import Path
from shutil import ignore_patterns, move, copy2, copystat from shutil import ignore_patterns, move, copy2, copystat
from stat import S_IWUSR as OWNER_WRITE_PERMISSION from stat import S_IWUSR as OWNER_WRITE_PERMISSION
@ -54,7 +54,7 @@ class Command(ScrapyCommand):
return True return True
return False return False
def _copytree(self, src, dst): def _copytree(self, src: Path, dst: Path):
""" """
Since the original function always creates the directory, to resolve Since the original function always creates the directory, to resolve
the issue a new function had to be created. It's a simple copy and the issue a new function had to be created. It's a simple copy and
@ -64,19 +64,19 @@ class Command(ScrapyCommand):
https://github.com/scrapy/scrapy/pull/2005 https://github.com/scrapy/scrapy/pull/2005
""" """
ignore = IGNORE ignore = IGNORE
names = os.listdir(src) names = [x.name for x in src.iterdir()]
ignored_names = ignore(src, names) ignored_names = ignore(src, names)
if not os.path.exists(dst): if not dst.exists():
os.makedirs(dst) dst.mkdir(parents=True)
for name in names: for name in names:
if name in ignored_names: if name in ignored_names:
continue continue
srcname = os.path.join(src, name) srcname = src / name
dstname = os.path.join(dst, name) dstname = dst / name
if os.path.isdir(srcname): if srcname.is_dir():
self._copytree(srcname, dstname) self._copytree(srcname, dstname)
else: else:
copy2(srcname, dstname) copy2(srcname, dstname)
@ -90,36 +90,36 @@ class Command(ScrapyCommand):
raise UsageError() raise UsageError()
project_name = args[0] project_name = args[0]
project_dir = args[0]
if len(args) == 2: if len(args) == 2:
project_dir = args[1] project_dir = Path(args[1])
else:
project_dir = Path(args[0])
if exists(join(project_dir, 'scrapy.cfg')): if (project_dir / 'scrapy.cfg').exists():
self.exitcode = 1 self.exitcode = 1
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}') print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}')
return return
if not self._is_valid_name(project_name): if not self._is_valid_name(project_name):
self.exitcode = 1 self.exitcode = 1
return return
self._copytree(self.templates_dir, abspath(project_dir)) self._copytree(Path(self.templates_dir), project_dir.resolve())
move(join(project_dir, 'module'), join(project_dir, project_name)) move(project_dir / 'module', project_dir / project_name)
for paths in TEMPLATES_TO_RENDER: for paths in TEMPLATES_TO_RENDER:
path = join(*paths) tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths))
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print(f"New Scrapy project '{project_name}', using template directory " print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:") f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n") print(f" {project_dir.resolve()}\n")
print("You can start your first spider with:") print("You can start your first spider with:")
print(f" cd {project_dir}") print(f" cd {project_dir}")
print(" scrapy genspider example example.com") print(" scrapy genspider example example.com")
@property @property
def templates_dir(self): def templates_dir(self) -> str:
return join( return str(Path(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'project' 'project'
) ))

5
scrapy/core/downloader/handlers/file.py
View File

@ -1,3 +1,5 @@
from pathlib import Path
from w3lib.url import file_uri_to_path from w3lib.url import file_uri_to_path
from scrapy.responsetypes import responsetypes from scrapy.responsetypes import responsetypes
@ -10,7 +12,6 @@ class FileDownloadHandler:
@defers @defers
def download_request(self, request, spider): def download_request(self, request, spider):
filepath = file_uri_to_path(request.url) filepath = file_uri_to_path(request.url)
with open(filepath, 'rb') as fo: body = Path(filepath).read_bytes()
body = fo.read()
respcls = responsetypes.from_args(filename=filepath, body=body) respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body) return respcls(url=request.url, body=body)

19
scrapy/core/scheduler.py
View File

@ -1,8 +1,7 @@
import json import json
import logging import logging
import os
from abc import abstractmethod from abc import abstractmethod
from os.path import exists, join from pathlib import Path
from typing import Optional, Type, TypeVar from typing import Optional, Type, TypeVar
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
@ -324,19 +323,19 @@ class Scheduler(BaseScheduler):
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
""" Return a folder name to keep disk queue state at """ """ Return a folder name to keep disk queue state at """
if jobdir is not None: if jobdir is not None:
dqdir = join(jobdir, 'requests.queue') dqdir = Path(jobdir, 'requests.queue')
if not exists(dqdir): if not dqdir.exists():
os.makedirs(dqdir) dqdir.mkdir(parents=True)
return dqdir return str(dqdir)
return None return None
def _read_dqs_state(self, dqdir: str) -> list: def _read_dqs_state(self, dqdir: str) -> list:
path = join(dqdir, 'active.json') path = Path(dqdir, 'active.json')
if not exists(path): if not path.exists():
return [] return []
with open(path) as f: with path.open() as f:
return json.load(f) return json.load(f)
def _write_dqs_state(self, dqdir: str, state: list) -> None: def _write_dqs_state(self, dqdir: str, state: list) -> None:
with open(join(dqdir, 'active.json'), 'w') as f: with Path(dqdir, 'active.json').open('w') as f:
json.dump(state, f) json.dump(state, f)

8
scrapy/crawler.py
View File

@ -1,7 +1,10 @@
from __future__ import annotations
import logging import logging
import pprint import pprint
import signal import signal
import warnings import warnings
from typing import TYPE_CHECKING
from twisted.internet import defer from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement from zope.interface.exceptions import DoesNotImplement
@ -33,6 +36,9 @@ from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import install_reactor, verify_installed_reactor from scrapy.utils.reactor import install_reactor, verify_installed_reactor
if TYPE_CHECKING:
from scrapy.utils.request import RequestFingerprinter
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -72,7 +78,7 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER']) lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self) self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance( self.request_fingerprinter: RequestFingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']), load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
settings=self.settings, settings=self.settings,
crawler=self, crawler=self,

4
scrapy/dupefilters.py
View File

@ -1,5 +1,5 @@
import logging import logging
import os from pathlib import Path
from typing import Optional, Set, Type, TypeVar from typing import Optional, Set, Type, TypeVar
from warnings import warn from warnings import warn
@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug self.debug = debug
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
if path: if path:
self.file = open(os.path.join(path, 'requests.seen'), 'a+') self.file = Path(path, 'requests.seen').open('a+')
self.file.seek(0) self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file) self.fingerprints.update(x.rstrip() for x in self.file)

16
scrapy/extensions/feedexport.py
View File

@ -5,13 +5,13 @@ See documentation in docs/topics/feed-exports.rst
""" """
import logging import logging
import os
import re import re
import sys import sys
import warnings import warnings
from datetime import datetime from datetime import datetime
from pathlib import Path
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from typing import Any, Callable, Optional, Tuple, Union from typing import IO, Any, Callable, Optional, Tuple, Union
from urllib.parse import unquote, urlparse from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads from twisted.internet import defer, threads
@ -101,7 +101,7 @@ class BlockingFeedStorage:
def open(self, spider): def open(self, spider):
path = spider.crawler.settings['FEED_TEMPDIR'] path = spider.crawler.settings['FEED_TEMPDIR']
if path and not os.path.isdir(path): if path and not Path(path).is_dir():
raise OSError('Not a Directory: ' + str(path)) raise OSError('Not a Directory: ' + str(path))
return NamedTemporaryFile(prefix='feed-', dir=path) return NamedTemporaryFile(prefix='feed-', dir=path)
@ -141,11 +141,11 @@ class FileFeedStorage:
feed_options = feed_options or {} feed_options = feed_options or {}
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
def open(self, spider): def open(self, spider) -> IO[Any]:
dirname = os.path.dirname(self.path) dirname = Path(self.path).parent
if dirname and not os.path.exists(dirname): if dirname and not dirname.exists():
os.makedirs(dirname) dirname.mkdir(parents=True)
return open(self.path, self.write_mode) return Path(self.path).open(self.write_mode)
def store(self, file): def store(self, file):
file.close() file.close()

48
scrapy/extensions/httpcache.py
View File

@ -1,16 +1,18 @@
import gzip import gzip
import logging import logging
import os
import pickle import pickle
from email.utils import mktime_tz, parsedate_tz from email.utils import mktime_tz, parsedate_tz
from importlib import import_module from importlib import import_module
from pathlib import Path
from time import time from time import time
from weakref import WeakKeyDictionary from weakref import WeakKeyDictionary
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response from scrapy.http import Headers, Response
from scrapy.http.request import Request
from scrapy.responsetypes import responsetypes from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.python import to_bytes, to_unicode
@ -221,9 +223,9 @@ class DbmCacheStorage:
self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
self.db = None self.db = None
def open_spider(self, spider): def open_spider(self, spider: Spider):
dbpath = os.path.join(self.cachedir, f'{spider.name}.db') dbpath = Path(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c') self.db = self.dbmodule.open(str(dbpath), 'c')
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
@ -277,7 +279,7 @@ class FilesystemCacheStorage:
self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
self._open = gzip.open if self.use_gzip else open self._open = gzip.open if self.use_gzip else open
def open_spider(self, spider): def open_spider(self, spider: Spider):
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider}) extra={'spider': spider})
@ -286,15 +288,15 @@ class FilesystemCacheStorage:
def close_spider(self, spider): def close_spider(self, spider):
pass pass
def retrieve_response(self, spider, request): def retrieve_response(self, spider: Spider, request: Request):
"""Return response if present in cache, or None otherwise.""" """Return response if present in cache, or None otherwise."""
metadata = self._read_meta(spider, request) metadata = self._read_meta(spider, request)
if metadata is None: if metadata is None:
return # not cached return # not cached
rpath = self._get_request_path(spider, request) rpath = self._get_request_path(spider, request)
with self._open(os.path.join(rpath, 'response_body'), 'rb') as f: with self._open(rpath / 'response_body', 'rb') as f:
body = f.read() body = f.read()
with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f: with self._open(rpath / 'response_headers', 'rb') as f:
rawheaders = f.read() rawheaders = f.read()
url = metadata.get('response_url') url = metadata.get('response_url')
status = metadata['status'] status = metadata['status']
@ -303,11 +305,11 @@ class FilesystemCacheStorage:
response = respcls(url=url, headers=headers, status=status, body=body) response = respcls(url=url, headers=headers, status=status, body=body)
return response return response
def store_response(self, spider, request, response): def store_response(self, spider: Spider, request: Request, response):
"""Store the given response in the cache.""" """Store the given response in the cache."""
rpath = self._get_request_path(spider, request) rpath = self._get_request_path(spider, request)
if not os.path.exists(rpath): if not rpath.exists():
os.makedirs(rpath) rpath.mkdir(parents=True)
metadata = { metadata = {
'url': request.url, 'url': request.url,
'method': request.method, 'method': request.method,
@ -315,29 +317,29 @@ class FilesystemCacheStorage:
'response_url': response.url, 'response_url': response.url,
'timestamp': time(), 'timestamp': time(),
} }
with self._open(os.path.join(rpath, 'meta'), 'wb') as f: with self._open(rpath / 'meta', 'wb') as f:
f.write(to_bytes(repr(metadata))) f.write(to_bytes(repr(metadata)))
with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: with self._open(rpath / 'pickled_meta', 'wb') as f:
pickle.dump(metadata, f, protocol=4) pickle.dump(metadata, f, protocol=4)
with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f: with self._open(rpath / 'response_headers', 'wb') as f:
f.write(headers_dict_to_raw(response.headers)) f.write(headers_dict_to_raw(response.headers))
with self._open(os.path.join(rpath, 'response_body'), 'wb') as f: with self._open(rpath / 'response_body', 'wb') as f:
f.write(response.body) f.write(response.body)
with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f: with self._open(rpath / 'request_headers', 'wb') as f:
f.write(headers_dict_to_raw(request.headers)) f.write(headers_dict_to_raw(request.headers))
with self._open(os.path.join(rpath, 'request_body'), 'wb') as f: with self._open(rpath / 'request_body', 'wb') as f:
f.write(request.body) f.write(request.body)
def _get_request_path(self, spider, request): def _get_request_path(self, spider: Spider, request: Request) -> Path:
key = self._fingerprinter.fingerprint(request).hex() key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key) return Path(self.cachedir, spider.name, key[0:2], key)
def _read_meta(self, spider, request): def _read_meta(self, spider: Spider, request: Request):
rpath = self._get_request_path(spider, request) rpath = self._get_request_path(spider, request)
metapath = os.path.join(rpath, 'pickled_meta') metapath = rpath / 'pickled_meta'
if not os.path.exists(metapath): if not metapath.exists():
return # not found return # not found
mtime = os.stat(metapath).st_mtime mtime = metapath.stat().st_mtime
if 0 < self.expiration_secs < time() - mtime: if 0 < self.expiration_secs < time() - mtime:
return # expired return # expired
with self._open(metapath, 'rb') as f: with self._open(metapath, 'rb') as f:

12
scrapy/extensions/spiderstate.py
View File

@ -1,5 +1,5 @@
import os
import pickle import pickle
from pathlib import Path
from scrapy import signals from scrapy import signals
from scrapy.exceptions import NotConfigured from scrapy.exceptions import NotConfigured
@ -25,16 +25,16 @@ class SpiderState:
def spider_closed(self, spider): def spider_closed(self, spider):
if self.jobdir: if self.jobdir:
with open(self.statefn, 'wb') as f: with Path(self.statefn).open('wb') as f:
pickle.dump(spider.state, f, protocol=4) pickle.dump(spider.state, f, protocol=4)
def spider_opened(self, spider): def spider_opened(self, spider):
if self.jobdir and os.path.exists(self.statefn): if self.jobdir and Path(self.statefn).exists():
with open(self.statefn, 'rb') as f: with Path(self.statefn).open('rb') as f:
spider.state = pickle.load(f) spider.state = pickle.load(f)
else: else:
spider.state = {} spider.state = {}
@property @property
def statefn(self): def statefn(self) -> str:
return os.path.join(self.jobdir, 'spider.state') return str(Path(self.jobdir, 'spider.state'))

4
scrapy/http/response/__init__.py
View File

@ -32,7 +32,7 @@ class Response(object_ref):
def __init__( def __init__(
self, self,
url, url: str,
status=200, status=200,
headers=None, headers=None,
body=b"", body=b"",
@ -75,7 +75,7 @@ class Response(object_ref):
def _get_url(self): def _get_url(self):
return self._url return self._url
def _set_url(self, url): def _set_url(self, url: str):
if isinstance(url, str): if isinstance(url, str):
self._url = url self._url = url
else: else:

41
scrapy/pipelines/files.py
View File

@ -13,6 +13,8 @@ from collections import defaultdict
from contextlib import suppress from contextlib import suppress
from ftplib import FTP from ftplib import FTP
from io import BytesIO from io import BytesIO
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse from urllib.parse import urlparse
from itemadapter import ItemAdapter from itemadapter import ItemAdapter
@ -39,41 +41,40 @@ class FileException(Exception):
class FSFilesStore: class FSFilesStore:
def __init__(self, basedir): def __init__(self, basedir: str):
if '://' in basedir: if '://' in basedir:
basedir = basedir.split('://', 1)[1] basedir = basedir.split('://', 1)[1]
self.basedir = basedir self.basedir = basedir
self._mkdir(self.basedir) self._mkdir(Path(self.basedir))
self.created_directories = defaultdict(set) self.created_directories: defaultdict[str, set[str]] = defaultdict(set)
def persist_file(self, path, buf, info, meta=None, headers=None): def persist_file(self, path: str, buf, info, meta=None, headers=None):
absolute_path = self._get_filesystem_path(path) absolute_path = self._get_filesystem_path(path)
self._mkdir(os.path.dirname(absolute_path), info) self._mkdir(absolute_path.parent, info)
with open(absolute_path, 'wb') as f: absolute_path.write_bytes(buf.getvalue())
f.write(buf.getvalue())
def stat_file(self, path, info): def stat_file(self, path: str, info):
absolute_path = self._get_filesystem_path(path) absolute_path = self._get_filesystem_path(path)
try: try:
last_modified = os.path.getmtime(absolute_path) last_modified = absolute_path.stat().st_mtime
except os.error: except os.error:
return {} return {}
with open(absolute_path, 'rb') as f: with absolute_path.open('rb') as f:
checksum = md5sum(f) checksum = md5sum(f)
return {'last_modified': last_modified, 'checksum': checksum} return {'last_modified': last_modified, 'checksum': checksum}
def _get_filesystem_path(self, path): def _get_filesystem_path(self, path: str) -> Path:
path_comps = path.split('/') path_comps = path.split('/')
return os.path.join(self.basedir, *path_comps) return Path(self.basedir, *path_comps)
def _mkdir(self, dirname, domain=None): def _mkdir(self, dirname: Path, domain: Optional[str] = None):
seen = self.created_directories[domain] if domain else set() seen = self.created_directories[domain] if domain else set()
if dirname not in seen: if str(dirname) not in seen:
if not os.path.exists(dirname): if not dirname.exists():
os.makedirs(dirname) dirname.mkdir(parents=True)
seen.add(dirname) seen.add(str(dirname))
class S3FilesStore: class S3FilesStore:
@ -374,8 +375,8 @@ class FilesPipeline(MediaPipeline):
store_uri = settings['FILES_STORE'] store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings) return cls(store_uri, settings=settings)
def _get_store(self, uri): def _get_store(self, uri: str):
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir
scheme = 'file' scheme = 'file'
else: else:
scheme = urlparse(uri).scheme scheme = urlparse(uri).scheme
@ -510,7 +511,7 @@ class FilesPipeline(MediaPipeline):
def file_path(self, request, response=None, info=None, *, item=None): def file_path(self, request, response=None, info=None, *, item=None):
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
media_ext = os.path.splitext(request.url)[1] media_ext = Path(request.url).suffix
# Handles empty and wild extensions by trying to guess the # Handles empty and wild extensions by trying to guess the
# mime type then extension or default to empty string otherwise # mime type then extension or default to empty string otherwise
if media_ext not in mimetypes.types_map: if media_ext not in mimetypes.types_map:

4
scrapy/settings/default_settings.py
View File

@ -15,7 +15,7 @@ Scrapy developers, if you add a setting here remember to:
import sys import sys
from importlib import import_module from importlib import import_module
from os.path import join, abspath, dirname from pathlib import Path
AJAXCRAWL_ENABLED = False AJAXCRAWL_ENABLED = False
@ -288,7 +288,7 @@ STATS_DUMP = True
STATSMAILER_RCPTS = [] STATSMAILER_RCPTS = []
TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve())
URLLENGTH_LIMIT = 2083 URLLENGTH_LIMIT = 2083

9
scrapy/spiders/__init__.py
View File

@ -3,14 +3,19 @@ Base class for Scrapy spiders
See documentation in docs/topics/spiders.rst See documentation in docs/topics/spiders.rst
""" """
from __future__ import annotations
import logging import logging
from typing import Optional from typing import TYPE_CHECKING, Optional
from scrapy import signals from scrapy import signals
from scrapy.http import Request from scrapy.http import Request
from scrapy.utils.trackref import object_ref from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class Spider(object_ref): class Spider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this """Base class for scrapy spiders. All spiders must inherit from this
@ -49,7 +54,7 @@ class Spider(object_ref):
spider._set_crawler(crawler) spider._set_crawler(crawler)
return spider return spider
def _set_crawler(self, crawler): def _set_crawler(self, crawler: Crawler):
self.crawler = crawler self.crawler = crawler
self.settings = crawler.settings self.settings = crawler.settings
crawler.signals.connect(self.close, signals.spider_closed) crawler.signals.connect(self.close, signals.spider_closed)

11
scrapy/squeues.py
View File

@ -3,8 +3,9 @@ Scheduler queues
""" """
import marshal import marshal
import os
import pickle import pickle
from os import PathLike
from pathlib import Path
from queuelib import queue from queuelib import queue
@ -16,10 +17,10 @@ def _with_mkdir(queue_class):
class DirectoriesCreated(queue_class): class DirectoriesCreated(queue_class):
def __init__(self, path, *args, **kwargs): def __init__(self, path: str | PathLike[str], *args, **kwargs):
dirname = os.path.dirname(path) dirname = Path(path).parent
if not os.path.exists(dirname): if not dirname.exists():
os.makedirs(dirname, exist_ok=True) dirname.mkdir(parents=True, exist_ok=True)
super().__init__(path, *args, **kwargs) super().__init__(path, *args, **kwargs)
return DirectoriesCreated return DirectoriesCreated

32
scrapy/utils/conf.py
View File

@ -4,6 +4,8 @@ import sys
import warnings import warnings
from configparser import ConfigParser from configparser import ConfigParser
from operator import itemgetter from operator import itemgetter
from pathlib import Path
from typing import Optional
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
@ -65,17 +67,17 @@ def arglist_to_dict(arglist):
return dict(x.split('=', 1) for x in arglist) return dict(x.split('=', 1) for x in arglist)
def closest_scrapy_cfg(path='.', prevpath=None): def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str:
"""Return the path to the closest scrapy.cfg file by traversing the current """Return the path to the closest scrapy.cfg file by traversing the current
directory and its parents directory and its parents
""" """
if path == prevpath: if prevpath is not None and str(path) == str(prevpath):
return '' return ''
path = os.path.abspath(path) path = Path(path).resolve()
cfgfile = os.path.join(path, 'scrapy.cfg') cfgfile = path / 'scrapy.cfg'
if os.path.exists(cfgfile): if cfgfile.exists():
return cfgfile return str(cfgfile)
return closest_scrapy_cfg(os.path.dirname(path), path) return closest_scrapy_cfg(path.parent, path)
def init_env(project='default', set_syspath=True): def init_env(project='default', set_syspath=True):
@ -88,7 +90,7 @@ def init_env(project='default', set_syspath=True):
os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project) os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)
closest = closest_scrapy_cfg() closest = closest_scrapy_cfg()
if closest: if closest:
projdir = os.path.dirname(closest) projdir = str(Path(closest).parent)
if set_syspath and projdir not in sys.path: if set_syspath and projdir not in sys.path:
sys.path.append(projdir) sys.path.append(projdir)
@ -101,13 +103,13 @@ def get_config(use_closest=True):
return cfg return cfg
def get_sources(use_closest=True): def get_sources(use_closest=True) -> list[str]:
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config') xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser()
sources = [ sources = [
'/etc/scrapy.cfg', '/etc/scrapy.cfg',
r'c:\scrapy\scrapy.cfg', r'c:\scrapy\scrapy.cfg',
xdg_config_home + '/scrapy.cfg', str(Path(xdg_config_home) / 'scrapy.cfg'),
os.path.expanduser('~/.scrapy.cfg'), str(Path('~/.scrapy.cfg').expanduser()),
] ]
if use_closest: if use_closest:
sources.append(closest_scrapy_cfg()) sources.append(closest_scrapy_cfg())
@ -129,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings):
return out return out
def feed_process_params_from_cli(settings, output, output_format=None, def feed_process_params_from_cli(settings, output: list[str], output_format=None,
overwrite_output=None): overwrite_output: Optional[list[str]] = None):
""" """
Receives feed export params (from the 'crawl' or 'runspider' commands), Receives feed export params (from the 'crawl' or 'runspider' commands),
checks for inconsistencies in their quantities and returns a dictionary checks for inconsistencies in their quantities and returns a dictionary
@ -180,7 +182,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
feed_uri, feed_format = element.rsplit(':', 1) feed_uri, feed_format = element.rsplit(':', 1)
except ValueError: except ValueError:
feed_uri = element feed_uri = element
feed_format = os.path.splitext(element)[1].replace('.', '') feed_format = Path(element).suffix.replace('.', '')
else: else:
if feed_uri == '-': if feed_uri == '-':
feed_uri = 'stdout:' feed_uri = 'stdout:'

6
scrapy/utils/job.py
View File

@ -1,4 +1,4 @@
import os from pathlib import Path
from typing import Optional from typing import Optional
from scrapy.settings import BaseSettings from scrapy.settings import BaseSettings
@ -6,6 +6,6 @@ from scrapy.settings import BaseSettings
def job_dir(settings: BaseSettings) -> Optional[str]: def job_dir(settings: BaseSettings) -> Optional[str]:
path = settings['JOBDIR'] path = settings['JOBDIR']
if path and not os.path.exists(path): if path and not Path(path).exists():
os.makedirs(path) Path(path).mkdir(parents=True)
return path return path

29
scrapy/utils/project.py
View File

@ -2,7 +2,7 @@ import os
import warnings import warnings
from importlib import import_module from importlib import import_module
from os.path import join, dirname, abspath, isabs, exists from pathlib import Path
from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
from scrapy.settings import Settings from scrapy.settings import Settings
@ -25,36 +25,37 @@ def inside_project():
return bool(closest_scrapy_cfg()) return bool(closest_scrapy_cfg())
def project_data_dir(project='default'): def project_data_dir(project='default') -> str:
"""Return the current project data dir, creating it if it doesn't exist""" """Return the current project data dir, creating it if it doesn't exist"""
if not inside_project(): if not inside_project():
raise NotConfigured("Not inside a project") raise NotConfigured("Not inside a project")
cfg = get_config() cfg = get_config()
if cfg.has_option(DATADIR_CFG_SECTION, project): if cfg.has_option(DATADIR_CFG_SECTION, project):
d = cfg.get(DATADIR_CFG_SECTION, project) d = Path(cfg.get(DATADIR_CFG_SECTION, project))
else: else:
scrapy_cfg = closest_scrapy_cfg() scrapy_cfg = closest_scrapy_cfg()
if not scrapy_cfg: if not scrapy_cfg:
raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir") raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
d = abspath(join(dirname(scrapy_cfg), '.scrapy')) d = (Path(scrapy_cfg).parent / '.scrapy').resolve()
if not exists(d): if not d.exists():
os.makedirs(d) d.mkdir(parents=True)
return d return str(d)
def data_path(path, createdir=False): def data_path(path: str, createdir=False) -> str:
""" """
Return the given path joined with the .scrapy data directory. Return the given path joined with the .scrapy data directory.
If given an absolute path, return it unmodified. If given an absolute path, return it unmodified.
""" """
if not isabs(path): path_obj = Path(path)
if not path_obj.is_absolute():
if inside_project(): if inside_project():
path = join(project_data_dir(), path) path_obj = Path(project_data_dir(), path)
else: else:
path = join('.scrapy', path) path_obj = Path('.scrapy', path)
if createdir and not exists(path): if createdir and not path_obj.exists():
os.makedirs(path) path_obj.mkdir(parents=True)
return path return str(path_obj)
def get_project_settings(): def get_project_settings():

2
scrapy/utils/request.py
View File

@ -264,7 +264,7 @@ class RequestFingerprinter:
f'and \'VERSION\'.' f'and \'VERSION\'.'
) )
def fingerprint(self, request): def fingerprint(self, request: Request):
return self._fingerprint(request) return self._fingerprint(request)

12
scrapy/utils/template.py
View File

@ -1,23 +1,21 @@
"""Helper functions for working with templates""" """Helper functions for working with templates"""
import os
import re import re
import string import string
from pathlib import Path
def render_templatefile(path, **kwargs): def render_templatefile(path: str, **kwargs):
with open(path, 'rb') as fp: raw = Path(path).read_text('utf8')
raw = fp.read().decode('utf8')
content = string.Template(raw).substitute(**kwargs) content = string.Template(raw).substitute(**kwargs)
render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
if path.endswith('.tmpl'): if path.endswith('.tmpl'):
os.rename(path, render_path) Path(path).rename(render_path)
with open(render_path, 'wb') as fp: Path(render_path).write_text(content, 'utf8')
fp.write(content.encode('utf8'))
CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')

5
scrapy/utils/test.py
View File

@ -4,6 +4,7 @@ This module contains some assorted functions used in tests
import asyncio import asyncio
import os import os
from pathlib import Path
from posixpath import split from posixpath import split
from unittest import mock from unittest import mock
@ -71,11 +72,11 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
return runner.create_crawler(spidercls or Spider) return runner.create_crawler(spidercls or Spider)
def get_pythonpath(): def get_pythonpath() -> str:
"""Return a PYTHONPATH suitable to use in processes so that they find this """Return a PYTHONPATH suitable to use in processes so that they find this
installation of Scrapy""" installation of Scrapy"""
scrapy_path = import_module('scrapy').__path__[0] scrapy_path = import_module('scrapy').__path__[0]
return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '') return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '')
def get_testenv(): def get_testenv():

5
setup.py
View File

@ -1,10 +1,9 @@
from os.path import dirname, join from pathlib import Path
from pkg_resources import parse_version from pkg_resources import parse_version
from setuptools import setup, find_packages, __version__ as setuptools_version from setuptools import setup, find_packages, __version__ as setuptools_version
with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip()
version = f.read().decode('ascii').strip()
def has_environment_marker_platform_impl_support(): def has_environment_marker_platform_impl_support():

17
tests/__init__.py
View File

@ -6,6 +6,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests
import os import os
import socket import socket
from pathlib import Path
# ignore system-wide proxies for tests # ignore system-wide proxies for tests
# which would send requests to a totally unsuspecting server # which would send requests to a totally unsuspecting server
@ -16,14 +17,12 @@ os.environ['ftp_proxy'] = ''
# Absolutize paths to coverage config and output file because tests that # Absolutize paths to coverage config and output file because tests that
# spawn subprocesses also changes current working directory. # spawn subprocesses also changes current working directory.
_sourceroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) _sourceroot = Path(__file__).resolve().parent.parent
if 'COV_CORE_CONFIG' in os.environ: if 'COV_CORE_CONFIG' in os.environ:
os.environ['COVERAGE_FILE'] = os.path.join(_sourceroot, '.coverage') os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage')
os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot, os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG'])
os.environ['COV_CORE_CONFIG'])
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data')
'sample_data')
# In some environments accessing a non-existing host doesn't raise an # In some environments accessing a non-existing host doesn't raise an
@ -35,8 +34,6 @@ except socket.gaierror:
NON_EXISTING_RESOLVABLE = False NON_EXISTING_RESOLVABLE = False
def get_testdata(*paths): def get_testdata(*paths: str) -> bytes:
"""Return test data""" """Return test data"""
path = os.path.join(tests_datadir, *paths) return Path(tests_datadir, *paths).read_bytes()
with open(path, 'rb') as f:
return f.read()

22
tests/keys/__init__.py
View File

@ -1,5 +1,5 @@
import os
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path
from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric import rsa
@ -22,21 +22,20 @@ from cryptography.x509.oid import NameOID
# https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate # https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate
def generate_keys(): def generate_keys():
folder = os.path.dirname(__file__) folder = Path(__file__).parent
key = rsa.generate_private_key( key = rsa.generate_private_key(
public_exponent=65537, public_exponent=65537,
key_size=2048, key_size=2048,
backend=default_backend(), backend=default_backend(),
) )
with open(os.path.join(folder, 'localhost.key'), "wb") as f: (folder / 'localhost.key').write_bytes(
f.write( key.private_bytes(
key.private_bytes( encoding=Encoding.PEM,
encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL,
format=PrivateFormat.TraditionalOpenSSL, encryption_algorithm=NoEncryption(),
encryption_algorithm=NoEncryption(), ),
) )
)
subject = issuer = Name( subject = issuer = Name(
[ [
@ -59,5 +58,4 @@ def generate_keys():
) )
.sign(key, SHA256(), default_backend()) .sign(key, SHA256(), default_backend())
) )
with open(os.path.join(folder, 'localhost.crt'), "wb") as f: (folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM))
f.write(cert.public_bytes(Encoding.PEM))

7
tests/mockserver.py
View File

@ -1,6 +1,5 @@
import argparse import argparse
import json import json
import os
import random import random
import sys import sys
from pathlib import Path from pathlib import Path
@ -253,7 +252,7 @@ class Root(resource.Resource):
self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
try: try:
from tests import tests_datadir from tests import tests_datadir
self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) self.putChild(b"files", File(str(Path(tests_datadir, 'test_site/files/'))))
except Exception: except Exception:
pass pass
self.putChild(b"redirect-to", RedirectTo()) self.putChild(b"redirect-to", RedirectTo())
@ -346,8 +345,8 @@ class MockFTPServer:
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
factory = ssl.DefaultOpenSSLContextFactory( factory = ssl.DefaultOpenSSLContextFactory(
os.path.join(os.path.dirname(__file__), keyfile), str(Path(__file__).parent / keyfile),
os.path.join(os.path.dirname(__file__), certfile), str(Path(__file__).parent / certfile),
) )
if cipher_string: if cipher_string:
ctx = factory.getContext() ctx = factory.getContext()

14
tests/test_cmdline/__init__.py
View File

@ -1,11 +1,11 @@
import json import json
import os
import pstats import pstats
import shutil import shutil
import sys import sys
import tempfile import tempfile
import unittest import unittest
from io import StringIO from io import StringIO
from pathlib import Path
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
from scrapy.utils.test import get_testenv from scrapy.utils.test import get_testenv
@ -36,17 +36,17 @@ class CmdlineTest(unittest.TestCase):
self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
def test_profiling(self): def test_profiling(self):
path = tempfile.mkdtemp() path = Path(tempfile.mkdtemp())
filename = os.path.join(path, 'res.prof') filename = path / 'res.prof'
try: try:
self._execute('version', '--profile', filename) self._execute('version', '--profile', str(filename))
self.assertTrue(os.path.exists(filename)) self.assertTrue(filename.exists())
out = StringIO() out = StringIO()
stats = pstats.Stats(filename, stream=out) stats = pstats.Stats(str(filename), stream=out)
stats.print_stats() stats.print_stats()
out.seek(0) out.seek(0)
stats = out.read() stats = out.read()
self.assertIn(os.path.join('scrapy', 'commands', 'version.py'), self.assertIn(str(Path('scrapy', 'commands', 'version.py')),
stats) stats)
self.assertIn('tottime', stats) self.assertIn('tottime', stats)
finally: finally:

4
tests/test_cmdline_crawl_with_pipeline/__init__.py
View File

@ -1,6 +1,6 @@
import os
import sys import sys
import unittest import unittest
from pathlib import Path
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
@ -8,7 +8,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase):
def _execute(self, spname): def _execute(self, spname):
args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname) args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)
cwd = os.path.dirname(os.path.abspath(__file__)) cwd = Path(__file__).resolve().parent
proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
proc.communicate() proc.communicate()
return proc.returncode return proc.returncode

9
tests/test_command_check.py
View File

@ -1,5 +1,3 @@
from os.path import join, abspath
from tests.test_commands import CommandTest from tests.test_commands import CommandTest
@ -10,11 +8,10 @@ class CheckCommandTest(CommandTest):
def setUp(self): def setUp(self):
super(CheckCommandTest, self).setUp() super(CheckCommandTest, self).setUp()
self.spider_name = 'check_spider' self.spider_name = 'check_spider'
self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve()
def _write_contract(self, contracts, parse_def): def _write_contract(self, contracts, parse_def):
with open(self.spider, 'w') as file: self.spider.write_text(f"""
file.write(f"""
import scrapy import scrapy
class CheckSpider(scrapy.Spider): class CheckSpider(scrapy.Spider):
@ -27,7 +24,7 @@ class CheckSpider(scrapy.Spider):
{contracts} {contracts}
\"\"\" \"\"\"
{parse_def} {parse_def}
""") """)
def _test_contract(self, contracts='', parse_def='pass'): def _test_contract(self, contracts='', parse_def='pass'):
self._write_contract(contracts, parse_def) self._write_contract(contracts, parse_def)

22
tests/test_command_parse.py
View File

@ -1,6 +1,6 @@
import os import os
import argparse import argparse
from os.path import join, abspath, isfile, exists from pathlib import Path
from twisted.internet import defer from twisted.internet import defer
from scrapy.commands import parse from scrapy.commands import parse
@ -23,9 +23,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.spider_name = 'parse_spider' self.spider_name = 'parse_spider'
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) (self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f"""
with open(fname, 'w') as f:
f.write(f"""
import scrapy import scrapy
from scrapy.linkextractors import LinkExtractor from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule from scrapy.spiders import CrawlSpider, Rule
@ -88,9 +86,7 @@ class MyBadCrawlSpider(CrawlSpider):
return [scrapy.Item(), dict(foo='bar')] return [scrapy.Item(), dict(foo='bar')]
""") """)
fname = abspath(join(self.proj_mod_path, 'pipelines.py')) (self.proj_mod_path / 'pipelines.py').write_text("""
with open(fname, 'w') as f:
f.write("""
import logging import logging
class MyPipeline: class MyPipeline:
@ -101,8 +97,7 @@ class MyPipeline:
return item return item
""") """)
fname = abspath(join(self.proj_mod_path, 'settings.py')) with (self.proj_mod_path / 'settings.py').open("a") as f:
with open(fname, 'a') as f:
f.write(f""" f.write(f"""
ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
""") """)
@ -234,7 +229,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
correct format containing correct data in it. correct format containing correct data in it.
""" """
file_name = 'data.json' file_name = 'data.json'
file_path = join(self.proj_path, file_name) file_path = Path(self.proj_path, file_name)
yield self.execute([ yield self.execute([
'--spider', self.spider_name, '--spider', self.spider_name,
'-c', 'parse', '-c', 'parse',
@ -242,12 +237,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.url('/html') self.url('/html')
]) ])
self.assertTrue(exists(file_path)) self.assertTrue(file_path.exists())
self.assertTrue(isfile(file_path)) self.assertTrue(file_path.is_file())
content = '[\n{},\n{"foo": "bar"}\n]' content = '[\n{},\n{"foo": "bar"}\n]'
with open(file_path, 'r') as f: self.assertEqual(file_path.read_text(), content)
self.assertEqual(f.read(), content)
def test_parse_add_options(self): def test_parse_add_options(self):
command = parse.Command() command = parse.Command()

6
tests/test_command_shell.py
View File

@ -1,4 +1,4 @@
from os.path import join from pathlib import Path
from twisted.trial import unittest from twisted.trial import unittest
from twisted.internet import defer from twisted.internet import defer
@ -96,8 +96,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def test_local_file(self): def test_local_file(self):
filepath = join(tests_datadir, 'test_site', 'index.html') filepath = Path(tests_datadir, 'test_site', 'index.html')
_, out, _ = yield self.execute([filepath, '-c', 'item']) _, out, _ = yield self.execute([str(filepath), '-c', 'item'])
assert b'{}' in out assert b'{}' in out
@defer.inlineCallbacks @defer.inlineCallbacks

199
tests/test_commands.py
View File

@ -9,12 +9,12 @@ import sys
import tempfile import tempfile
from contextlib import contextmanager from contextlib import contextmanager
from itertools import chain from itertools import chain
from os.path import exists, join, abspath, getmtime
from pathlib import Path from pathlib import Path
from shutil import rmtree, copytree from shutil import rmtree, copytree
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
from tempfile import mkdtemp from tempfile import mkdtemp
from threading import Timer from threading import Timer
from typing import Generator, Optional
from unittest import skipIf from unittest import skipIf
from pytest import mark from pytest import mark
@ -66,8 +66,8 @@ class ProjectTest(unittest.TestCase):
def setUp(self): def setUp(self):
self.temp_path = mkdtemp() self.temp_path = mkdtemp()
self.cwd = self.temp_path self.cwd = self.temp_path
self.proj_path = join(self.temp_path, self.project_name) self.proj_path = Path(self.temp_path, self.project_name)
self.proj_mod_path = join(self.proj_path, self.project_name) self.proj_mod_path = self.proj_path / self.project_name
self.env = get_testenv() self.env = get_testenv()
def tearDown(self): def tearDown(self):
@ -104,10 +104,10 @@ class ProjectTest(unittest.TestCase):
return p, to_unicode(stdout), to_unicode(stderr) return p, to_unicode(stdout), to_unicode(stderr)
def find_in_file(self, filename, regex): def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]:
"""Find first pattern occurrence in file""" """Find first pattern occurrence in file"""
pattern = re.compile(regex) pattern = re.compile(regex)
with open(filename, "r") as f: with Path(filename).open("r") as f:
for line in f: for line in f:
match = pattern.search(line) match = pattern.search(line)
if match is not None: if match is not None:
@ -122,13 +122,13 @@ class StartprojectTest(ProjectTest):
print(err, file=sys.stderr) print(err, file=sys.stderr)
self.assertEqual(p.returncode, 0) self.assertEqual(p.returncode, 0)
assert exists(join(self.proj_path, 'scrapy.cfg')) assert Path(self.proj_path, 'scrapy.cfg').exists()
assert exists(join(self.proj_path, 'testproject')) assert Path(self.proj_path, 'testproject').exists()
assert exists(join(self.proj_mod_path, '__init__.py')) assert Path(self.proj_mod_path, '__init__.py').exists()
assert exists(join(self.proj_mod_path, 'items.py')) assert Path(self.proj_mod_path, 'items.py').exists()
assert exists(join(self.proj_mod_path, 'pipelines.py')) assert Path(self.proj_mod_path, 'pipelines.py').exists()
assert exists(join(self.proj_mod_path, 'settings.py')) assert Path(self.proj_mod_path, 'settings.py').exists()
assert exists(join(self.proj_mod_path, 'spiders', '__init__.py')) assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists()
self.assertEqual(1, self.call('startproject', self.project_name)) self.assertEqual(1, self.call('startproject', self.project_name))
self.assertEqual(1, self.call('startproject', 'wrong---project---name')) self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
@ -138,13 +138,13 @@ class StartprojectTest(ProjectTest):
project_dir = mkdtemp() project_dir = mkdtemp()
self.assertEqual(0, self.call('startproject', self.project_name, project_dir)) self.assertEqual(0, self.call('startproject', self.project_name, project_dir))
assert exists(join(abspath(project_dir), 'scrapy.cfg')) assert Path(project_dir, 'scrapy.cfg').exists()
assert exists(join(abspath(project_dir), 'testproject')) assert Path(project_dir, 'testproject').exists()
assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) assert Path(project_dir, self.project_name, '__init__.py').exists()
assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) assert Path(project_dir, self.project_name, 'items.py').exists()
assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) assert Path(project_dir, self.project_name, 'pipelines.py').exists()
assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) assert Path(project_dir, self.project_name, 'settings.py').exists()
assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists()
self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))
@ -158,40 +158,42 @@ class StartprojectTest(ProjectTest):
def test_existing_project_dir(self): def test_existing_project_dir(self):
project_dir = mkdtemp() project_dir = mkdtemp()
project_name = self.project_name + '_existing' project_name = self.project_name + '_existing'
project_path = os.path.join(project_dir, project_name) project_path = Path(project_dir, project_name)
os.mkdir(project_path) project_path.mkdir()
p, out, err = self.proc('startproject', project_name, cwd=project_dir) p, out, err = self.proc('startproject', project_name, cwd=project_dir)
print(out) print(out)
print(err, file=sys.stderr) print(err, file=sys.stderr)
self.assertEqual(p.returncode, 0) self.assertEqual(p.returncode, 0)
assert exists(join(abspath(project_path), 'scrapy.cfg')) assert Path(project_path, 'scrapy.cfg').exists()
assert exists(join(abspath(project_path), project_name)) assert Path(project_path, project_name).exists()
assert exists(join(join(abspath(project_path), project_name), '__init__.py')) assert Path(project_path, project_name, '__init__.py').exists()
assert exists(join(join(abspath(project_path), project_name), 'items.py')) assert Path(project_path, project_name, 'items.py').exists()
assert exists(join(join(abspath(project_path), project_name), 'pipelines.py')) assert Path(project_path, project_name, 'pipelines.py').exists()
assert exists(join(join(abspath(project_path), project_name), 'settings.py')) assert Path(project_path, project_name, 'settings.py').exists()
assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py')) assert Path(project_path, project_name, 'spiders', '__init__.py').exists()
def get_permissions_dict(path, renamings=None, ignore=None): def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]:
def get_permissions(path): def get_permissions(path: Path) -> str:
return oct(os.stat(path).st_mode) return oct(path.stat().st_mode)
path_obj = Path(path)
renamings = renamings or tuple() renamings = renamings or tuple()
permissions_dict = { permissions_dict = {
'.': get_permissions(path), '.': get_permissions(path_obj),
} }
for root, dirs, files in os.walk(path): for root, dirs, files in os.walk(path_obj):
nodes = list(chain(dirs, files)) nodes = list(chain(dirs, files))
if ignore: if ignore:
ignored_names = ignore(root, nodes) ignored_names = ignore(root, nodes)
nodes = [node for node in nodes if node not in ignored_names] nodes = [node for node in nodes if node not in ignored_names]
for node in nodes: for node in nodes:
absolute_path = os.path.join(root, node) absolute_path = Path(root, node)
relative_path = os.path.relpath(absolute_path, path) relative_path = str(absolute_path.relative_to(path))
for search_string, replacement in renamings: for search_string, replacement in renamings:
relative_path = relative_path.replace( relative_path = relative_path.replace(
search_string, search_string,
@ -208,28 +210,27 @@ class StartprojectTemplatesTest(ProjectTest):
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.tmpl = join(self.temp_path, 'templates') self.tmpl = str(Path(self.temp_path, 'templates'))
self.tmpl_proj = join(self.tmpl, 'project') self.tmpl_proj = str(Path(self.tmpl, 'project'))
def test_startproject_template_override(self): def test_startproject_template_override(self):
copytree(join(scrapy.__path__[0], 'templates'), self.tmpl) copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl)
with open(join(self.tmpl_proj, 'root_template'), 'w'): Path(self.tmpl_proj, 'root_template').write_bytes(b"")
pass assert Path(self.tmpl_proj, 'root_template').exists()
assert exists(join(self.tmpl_proj, 'root_template'))
args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] args = ['--set', f'TEMPLATES_DIR={self.tmpl}']
p, out, err = self.proc('startproject', self.project_name, *args) p, out, err = self.proc('startproject', self.project_name, *args)
self.assertIn(f"New Scrapy project '{self.project_name}', " self.assertIn(f"New Scrapy project '{self.project_name}', "
"using template directory", out) "using template directory", out)
self.assertIn(self.tmpl_proj, out) self.assertIn(self.tmpl_proj, out)
assert exists(join(self.proj_path, 'root_template')) assert Path(self.proj_path, 'root_template').exists()
def test_startproject_permissions_from_writable(self): def test_startproject_permissions_from_writable(self):
"""Check that generated files have the right permissions when the """Check that generated files have the right permissions when the
template folder has the same permissions as in the project, i.e. template folder has the same permissions as in the project, i.e.
everything is writable.""" everything is writable."""
scrapy_path = scrapy.__path__[0] scrapy_path = scrapy.__path__[0]
project_template = os.path.join(scrapy_path, 'templates', 'project') project_template = Path(scrapy_path, 'templates', 'project')
project_name = 'startproject1' project_name = 'startproject1'
renamings = ( renamings = (
('module', project_name), ('module', project_name),
@ -255,7 +256,7 @@ class StartprojectTemplatesTest(ProjectTest):
) )
process.wait() process.wait()
project_dir = os.path.join(destination, project_name) project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir) actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions) self.assertEqual(actual_permissions, expected_permissions)
@ -268,8 +269,8 @@ class StartprojectTemplatesTest(ProjectTest):
See https://github.com/scrapy/scrapy/pull/4604 See https://github.com/scrapy/scrapy/pull/4604
""" """
scrapy_path = scrapy.__path__[0] scrapy_path = scrapy.__path__[0]
templates_dir = os.path.join(scrapy_path, 'templates') templates_dir = Path(scrapy_path, 'templates')
project_template = os.path.join(templates_dir, 'project') project_template = Path(templates_dir, 'project')
project_name = 'startproject2' project_name = 'startproject2'
renamings = ( renamings = (
('module', project_name), ('module', project_name),
@ -281,16 +282,16 @@ class StartprojectTemplatesTest(ProjectTest):
IGNORE, IGNORE,
) )
def _make_read_only(path): def _make_read_only(path: Path):
current_permissions = os.stat(path).st_mode current_permissions = path.stat().st_mode
os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION) path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION)
read_only_templates_dir = str(Path(mkdtemp()) / 'templates') read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
copytree(templates_dir, read_only_templates_dir) copytree(templates_dir, read_only_templates_dir)
for root, dirs, files in os.walk(read_only_templates_dir): for root, dirs, files in os.walk(read_only_templates_dir):
for node in chain(dirs, files): for node in chain(dirs, files):
_make_read_only(os.path.join(root, node)) _make_read_only(Path(root, node))
destination = mkdtemp() destination = mkdtemp()
process = subprocess.Popen( process = subprocess.Popen(
@ -308,7 +309,7 @@ class StartprojectTemplatesTest(ProjectTest):
) )
process.wait() process.wait()
project_dir = os.path.join(destination, project_name) project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir) actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions) self.assertEqual(actual_permissions, expected_permissions)
@ -317,7 +318,7 @@ class StartprojectTemplatesTest(ProjectTest):
"""Check that pre-existing folders and files in the destination folder """Check that pre-existing folders and files in the destination folder
do not see their permissions modified.""" do not see their permissions modified."""
scrapy_path = scrapy.__path__[0] scrapy_path = scrapy.__path__[0]
project_template = os.path.join(scrapy_path, 'templates', 'project') project_template = Path(scrapy_path, 'templates', 'project')
project_name = 'startproject3' project_name = 'startproject3'
renamings = ( renamings = (
('module', project_name), ('module', project_name),
@ -330,7 +331,7 @@ class StartprojectTemplatesTest(ProjectTest):
) )
destination = mkdtemp() destination = mkdtemp()
project_dir = os.path.join(destination, project_name) project_dir = Path(destination, project_name)
existing_nodes = { existing_nodes = {
oct(permissions)[2:] + extension: permissions oct(permissions)[2:] + extension: permissions
@ -339,10 +340,9 @@ class StartprojectTemplatesTest(ProjectTest):
0o444, 0o555, 0o644, 0o666, 0o755, 0o777, 0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
) )
} }
os.mkdir(project_dir) project_dir.mkdir()
project_dir_path = Path(project_dir)
for node, permissions in existing_nodes.items(): for node, permissions in existing_nodes.items():
path = project_dir_path / node path = project_dir / node
if node.endswith('.d'): if node.endswith('.d'):
path.mkdir(mode=permissions) path.mkdir(mode=permissions)
else: else:
@ -378,7 +378,7 @@ class StartprojectTemplatesTest(ProjectTest):
os.umask(cur_mask) os.umask(cur_mask)
scrapy_path = scrapy.__path__[0] scrapy_path = scrapy.__path__[0]
project_template = os.path.join( project_template = Path(
scrapy_path, scrapy_path,
'templates', 'templates',
'project' 'project'
@ -409,7 +409,7 @@ class StartprojectTemplatesTest(ProjectTest):
) )
process.wait() process.wait()
project_dir = os.path.join(destination, project_name) project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir) actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions) self.assertEqual(actual_permissions, expected_permissions)
@ -420,7 +420,7 @@ class CommandTest(ProjectTest):
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.call('startproject', self.project_name) self.call('startproject', self.project_name)
self.cwd = join(self.temp_path, self.project_name) self.cwd = Path(self.temp_path, self.project_name)
self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings'
@ -429,10 +429,10 @@ class GenspiderCommandTest(CommandTest):
def test_arguments(self): def test_arguments(self):
# only pass one argument. spider script shouldn't be created # only pass one argument. spider script shouldn't be created
self.assertEqual(2, self.call('genspider', 'test_name')) self.assertEqual(2, self.call('genspider', 'test_name'))
assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
# pass two arguments <name> <domain>. spider script should be created # pass two arguments <name> <domain>. spider script should be created
self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) self.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))
assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
def test_template(self, tplname='crawl'): def test_template(self, tplname='crawl'):
args = [f'--template={tplname}'] if tplname else [] args = [f'--template={tplname}'] if tplname else []
@ -440,11 +440,11 @@ class GenspiderCommandTest(CommandTest):
spmodule = f"{self.project_name}.spiders.{spname}" spmodule = f"{self.project_name}.spiders.{spname}"
p, out, err = self.proc('genspider', spname, 'test.com', *args) p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out)
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists())
modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
p, out, err = self.proc('genspider', spname, 'test.com', *args) p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn(f"Spider {spname!r} already exists in module", out) self.assertIn(f"Spider {spname!r} already exists in module", out)
modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before) self.assertEqual(modify_time_after, modify_time_before)
def test_template_basic(self): def test_template_basic(self):
@ -465,37 +465,37 @@ class GenspiderCommandTest(CommandTest):
def test_same_name_as_project(self): def test_same_name_as_project(self):
self.assertEqual(2, self.call('genspider', self.project_name)) self.assertEqual(2, self.call('genspider', self.project_name))
assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py')) assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists()
def test_same_filename_as_existing_spider(self, force=False): def test_same_filename_as_existing_spider(self, force=False):
file_name = 'example' file_name = 'example'
file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py') file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py')
self.assertEqual(0, self.call('genspider', file_name, 'example.com')) self.assertEqual(0, self.call('genspider', file_name, 'example.com'))
assert exists(file_path) assert file_path.exists()
# change name of spider but not its file name # change name of spider but not its file name
with open(file_path, 'r+') as spider_file: with file_path.open('r+') as spider_file:
file_data = spider_file.read() file_data = spider_file.read()
file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
spider_file.seek(0) spider_file.seek(0)
spider_file.write(file_data) spider_file.write(file_data)
spider_file.truncate() spider_file.truncate()
modify_time_before = getmtime(file_path) modify_time_before = file_path.stat().st_mtime
file_contents_before = file_data file_contents_before = file_data
if force: if force:
p, out, err = self.proc('genspider', '--force', file_name, 'example.com') p, out, err = self.proc('genspider', '--force', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out)
modify_time_after = getmtime(file_path) modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before) self.assertNotEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read() file_contents_after = file_path.read_text()
self.assertNotEqual(file_contents_after, file_contents_before) self.assertNotEqual(file_contents_after, file_contents_before)
else: else:
p, out, err = self.proc('genspider', file_name, 'example.com') p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"{file_path} already exists", out) self.assertIn(f"{file_path} already exists", out)
modify_time_after = getmtime(file_path) modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before) self.assertEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read() file_contents_after = file_path.read_text()
self.assertEqual(file_contents_after, file_contents_before) self.assertEqual(file_contents_after, file_contents_before)
def test_same_filename_as_existing_spider_force(self): def test_same_filename_as_existing_spider_force(self):
@ -504,11 +504,11 @@ class GenspiderCommandTest(CommandTest):
def test_url(self, url='test.com', domain="test.com"): def test_url(self, url='test.com', domain="test.com"):
self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) self.assertEqual(0, self.call('genspider', '--force', 'test_name', url))
self.assertEqual(domain, self.assertEqual(domain,
self.find_in_file(join(self.proj_mod_path, self.find_in_file(Path(self.proj_mod_path,
'spiders', 'test_name.py'), 'spiders', 'test_name.py'),
r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
self.assertEqual(f'http://{domain}/', self.assertEqual(f'http://{domain}/',
self.find_in_file(join(self.proj_mod_path, self.find_in_file(Path(self.proj_mod_path,
'spiders', 'test_name.py'), 'spiders', 'test_name.py'),
r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
@ -526,31 +526,31 @@ class GenspiderStandaloneCommandTest(ProjectTest):
def test_generate_standalone_spider(self): def test_generate_standalone_spider(self):
self.call('genspider', 'example', 'example.com') self.call('genspider', 'example', 'example.com')
assert exists(join(self.temp_path, 'example.py')) assert Path(self.temp_path, 'example.py').exists()
def test_same_name_as_existing_file(self, force=False): def test_same_name_as_existing_file(self, force=False):
file_name = 'example' file_name = 'example'
file_path = join(self.temp_path, file_name + '.py') file_path = Path(self.temp_path, file_name + '.py')
p, out, err = self.proc('genspider', file_name, 'example.com') p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out)
assert exists(file_path) assert file_path.exists()
modify_time_before = getmtime(file_path) modify_time_before = file_path.stat().st_mtime
file_contents_before = open(file_path, 'r').read() file_contents_before = file_path.read_text()
if force: if force:
# use different template to ensure contents were changed # use different template to ensure contents were changed
p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out)
modify_time_after = getmtime(file_path) modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before) self.assertNotEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read() file_contents_after = file_path.read_text()
self.assertNotEqual(file_contents_after, file_contents_before) self.assertNotEqual(file_contents_after, file_contents_before)
else: else:
p, out, err = self.proc('genspider', file_name, 'example.com') p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out) self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out)
modify_time_after = getmtime(file_path) modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before) self.assertEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read() file_contents_after = file_path.read_text()
self.assertEqual(file_contents_after, file_contents_before) self.assertEqual(file_contents_after, file_contents_before)
def test_same_name_as_existing_file_force(self): def test_same_name_as_existing_file_force(self):
@ -588,17 +588,16 @@ class BadSpider(scrapy.Spider):
""" """
@contextmanager @contextmanager
def _create_file(self, content, name=None): def _create_file(self, content, name=None) -> Generator[str, None, None]:
tmpdir = self.mktemp() tmpdir = Path(self.mktemp())
os.mkdir(tmpdir) tmpdir.mkdir()
if name: if name:
fname = abspath(join(tmpdir, name)) fname = (tmpdir / name).resolve()
else: else:
fname = abspath(join(tmpdir, self.spider_filename)) fname = (tmpdir / self.spider_filename).resolve()
with open(fname, 'w') as f: fname.write_text(content)
f.write(content)
try: try:
yield fname yield str(fname)
finally: finally:
rmtree(tmpdir) rmtree(tmpdir)
@ -747,12 +746,11 @@ class MySpider(scrapy.Spider):
) )
return [] return []
""" """
with open(os.path.join(self.cwd, "example.json"), "w") as f1: Path(self.cwd, "example.json").write_text("not empty")
f1.write("not empty")
args = ['-O', 'example.json'] args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args) log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
with open(os.path.join(self.cwd, "example.json")) as f2: with Path(self.cwd, "example.json").open() as f2:
first_line = f2.readline() first_line = f2.readline()
self.assertNotEqual(first_line, "not empty") self.assertNotEqual(first_line, "not empty")
@ -854,9 +852,7 @@ class ViewCommandTest(CommandTest):
class CrawlCommandTest(CommandTest): class CrawlCommandTest(CommandTest):
def crawl(self, code, args=()): def crawl(self, code, args=()):
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code)
with open(fname, 'w') as f:
f.write(code)
return self.proc('crawl', 'myspider', *args) return self.proc('crawl', 'myspider', *args)
def get_log(self, code, args=()): def get_log(self, code, args=()):
@ -908,12 +904,11 @@ class MySpider(scrapy.Spider):
) )
return [] return []
""" """
with open(os.path.join(self.cwd, "example.json"), "w") as f1: Path(self.cwd, "example.json").write_text("not empty")
f1.write("not empty")
args = ['-O', 'example.json'] args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args) log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
with open(os.path.join(self.cwd, "example.json")) as f2: with Path(self.cwd, "example.json").open() as f2:
first_line = f2.readline() first_line = f2.readline()
self.assertNotEqual(first_line, "not empty") self.assertNotEqual(first_line, "not empty")

32
tests/test_crawler.py
View File

@ -1,9 +1,9 @@
import logging import logging
import os
import platform import platform
import subprocess import subprocess
import sys import sys
import warnings import warnings
from pathlib import Path
from pytest import raises, mark from pytest import raises, mark
from twisted import version as twisted_version from twisted import version as twisted_version
@ -100,15 +100,14 @@ class CrawlerLoggingTestCase(unittest.TestCase):
assert get_scrapy_root_handler() is None assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self): def test_spider_custom_settings_log_level(self):
log_file = self.mktemp() log_file = Path(self.mktemp())
with open(log_file, 'wb') as fo: log_file.write_text('previous message\n', encoding='utf-8')
fo.write('previous message\n'.encode('utf-8'))
class MySpider(scrapy.Spider): class MySpider(scrapy.Spider):
name = 'spider' name = 'spider'
custom_settings = { custom_settings = {
'LOG_LEVEL': 'INFO', 'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file, 'LOG_FILE': str(log_file),
# settings to avoid extra warnings # settings to avoid extra warnings
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
@ -124,8 +123,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
logging.warning('warning message') logging.warning('warning message')
logging.error('error message') logging.error('error message')
with open(log_file, 'rb') as fo: logged = log_file.read_text(encoding='utf-8')
logged = fo.read().decode('utf-8')
self.assertIn('previous message', logged) self.assertIn('previous message', logged)
self.assertNotIn('debug message', logged) self.assertNotIn('debug message', logged)
@ -139,14 +137,13 @@ class CrawlerLoggingTestCase(unittest.TestCase):
self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
def test_spider_custom_settings_log_append(self): def test_spider_custom_settings_log_append(self):
log_file = self.mktemp() log_file = Path(self.mktemp())
with open(log_file, 'wb') as fo: log_file.write_text('previous message\n', encoding='utf-8')
fo.write('previous message\n'.encode('utf-8'))
class MySpider(scrapy.Spider): class MySpider(scrapy.Spider):
name = 'spider' name = 'spider'
custom_settings = { custom_settings = {
'LOG_FILE': log_file, 'LOG_FILE': str(log_file),
'LOG_FILE_APPEND': False, 'LOG_FILE_APPEND': False,
# disable telnet if not available to avoid an extra warning # disable telnet if not available to avoid an extra warning
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
@ -156,8 +153,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
get_crawler(MySpider) get_crawler(MySpider)
logging.debug('debug message') logging.debug('debug message')
with open(log_file, 'rb') as fo: logged = log_file.read_text(encoding='utf-8')
logged = fo.read().decode('utf-8')
self.assertNotIn('previous message', logged) self.assertNotIn('previous message', logged)
self.assertIn('debug message', logged) self.assertIn('debug message', logged)
@ -296,9 +292,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
class ScriptRunnerMixin: class ScriptRunnerMixin:
def run_script(self, script_name, *script_args): def run_script(self, script_name: str, *script_args):
script_path = os.path.join(self.script_dir, script_name) script_path = self.script_dir / script_name
args = [sys.executable, script_path] + list(script_args) args = [sys.executable, str(script_path)] + list(script_args)
p = subprocess.Popen(args, env=get_testenv(), p = subprocess.Popen(args, env=get_testenv(),
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate() stdout, stderr = p.communicate()
@ -306,7 +302,7 @@ class ScriptRunnerMixin:
class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess'
def test_simple(self): def test_simple(self):
log = self.run_script('simple.py') log = self.run_script('simple.py')
@ -463,7 +459,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner'
def test_response_ip_address(self): def test_response_ip_address(self):
log = self.run_script("ip_address.py") log = self.run_script("ip_address.py")

7
tests/test_dependencies.py
View File

@ -2,6 +2,7 @@ import os
import re import re
from configparser import ConfigParser from configparser import ConfigParser
from importlib import import_module from importlib import import_module
from pathlib import Path
from twisted import version as twisted_version from twisted import version as twisted_version
from twisted.trial import unittest from twisted.trial import unittest
@ -29,11 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase):
if not os.environ.get('_SCRAPY_PINNED', None): if not os.environ.get('_SCRAPY_PINNED', None):
self.skipTest('Not in a pinned environment') self.skipTest('Not in a pinned environment')
tox_config_file_path = os.path.join( tox_config_file_path = Path(__file__) / '..' / 'tox.ini'
os.path.dirname(__file__),
'..',
'tox.ini',
)
config_parser = ConfigParser() config_parser = ConfigParser()
config_parser.read(tox_config_file_path) config_parser.read(tox_config_file_path)
pattern = r'Twisted\[http2\]==([\d.]+)' pattern = r'Twisted\[http2\]==([\d.]+)'

69
tests/test_downloader_handlers.py
View File

@ -3,6 +3,7 @@ import os
import shutil import shutil
import sys import sys
import tempfile import tempfile
from pathlib import Path
from typing import Optional, Type from typing import Optional, Type
from unittest import mock, SkipTest from unittest import mock, SkipTest
@ -10,7 +11,6 @@ from testfixtures import LogCapture
from twisted.cred import checkers, credentials, portal from twisted.cred import checkers, credentials, portal
from twisted.internet import defer, error, reactor from twisted.internet import defer, error, reactor
from twisted.protocols.policies import WrappingFactory from twisted.protocols.policies import WrappingFactory
from twisted.python.filepath import FilePath
from twisted.trial import unittest from twisted.trial import unittest
from twisted.web import resource, server, static, util from twisted.web import resource, server, static, util
from twisted.web._newclient import ResponseFailed from twisted.web._newclient import ResponseFailed
@ -108,14 +108,13 @@ class LoadTestCase(unittest.TestCase):
class FileTestCase(unittest.TestCase): class FileTestCase(unittest.TestCase):
def setUp(self): def setUp(self):
self.tmpname = self.mktemp() self.tmpname = Path(self.mktemp() + '^')
with open(self.tmpname + '^', 'w') as f: Path(self.tmpname).write_text('0123456789')
f.write('0123456789')
handler = create_instance(FileDownloadHandler, None, get_crawler()) handler = create_instance(FileDownloadHandler, None, get_crawler())
self.download_request = handler.download_request self.download_request = handler.download_request
def tearDown(self): def tearDown(self):
os.unlink(self.tmpname + '^') self.tmpname.unlink()
def test_download(self): def test_download(self):
def _test(response): def _test(response):
@ -124,7 +123,7 @@ class FileTestCase(unittest.TestCase):
self.assertEqual(response.body, b'0123456789') self.assertEqual(response.body, b'0123456789')
self.assertEqual(response.protocol, None) self.assertEqual(response.protocol, None)
request = Request(path_to_file_uri(self.tmpname + '^')) request = Request(path_to_file_uri(str(self.tmpname)))
assert request.url.upper().endswith('%5E') assert request.url.upper().endswith('%5E')
return self.download_request(request, Spider('foo')).addCallback(_test) return self.download_request(request, Spider('foo')).addCallback(_test)
@ -223,10 +222,10 @@ class HttpTestCase(unittest.TestCase):
certfile = 'keys/localhost.crt' certfile = 'keys/localhost.crt'
def setUp(self): def setUp(self):
self.tmpname = self.mktemp() self.tmpname = Path(self.mktemp())
os.mkdir(self.tmpname) self.tmpname.mkdir()
FilePath(self.tmpname).child("file").setContent(b"0123456789") (self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(self.tmpname) r = static.File(str(self.tmpname))
r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
@ -626,10 +625,10 @@ class Https11CustomCiphers(unittest.TestCase):
certfile = 'keys/localhost.crt' certfile = 'keys/localhost.crt'
def setUp(self): def setUp(self):
self.tmpname = self.mktemp() self.tmpname = Path(self.mktemp())
os.mkdir(self.tmpname) self.tmpname.mkdir()
FilePath(self.tmpname).child("file").setContent(b"0123456789") (self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(self.tmpname) r = static.File(str(self.tmpname))
self.site = server.Site(r, timeout=None) self.site = server.Site(r, timeout=None)
self.host = 'localhost' self.host = 'localhost'
self.port = reactor.listenSSL( self.port = reactor.listenSSL(
@ -1002,16 +1001,15 @@ class BaseFTPTestCase(unittest.TestCase):
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
# setup dirs and test file # setup dirs and test file
self.directory = self.mktemp() self.directory = Path(self.mktemp())
os.mkdir(self.directory) self.directory.mkdir()
userdir = os.path.join(self.directory, self.username) userdir = self.directory / self.username
os.mkdir(userdir) userdir.mkdir()
fp = FilePath(userdir)
for filename, content in self.test_files: for filename, content in self.test_files:
fp.child(filename).setContent(content) (userdir / filename).write_bytes(content)
# setup server # setup server
realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory))
p = portal.Portal(realm) p = portal.Portal(realm)
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
users_checker.addUser(self.username, self.password) users_checker.addUser(self.username, self.password)
@ -1076,28 +1074,28 @@ class BaseFTPTestCase(unittest.TestCase):
def test_ftp_local_filename(self): def test_ftp_local_filename(self):
f, local_fname = tempfile.mkstemp() f, local_fname = tempfile.mkstemp()
local_fname = to_bytes(local_fname) fname_bytes = to_bytes(local_fname)
local_fname = Path(local_fname)
os.close(f) os.close(f)
meta = {"ftp_local_filename": local_fname} meta = {"ftp_local_filename": fname_bytes}
meta.update(self.req_meta) meta.update(self.req_meta)
request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt",
meta=meta) meta=meta)
d = self.download_handler.download_request(request, None) d = self.download_handler.download_request(request, None)
def _test(r): def _test(r):
self.assertEqual(r.body, local_fname) self.assertEqual(r.body, fname_bytes)
self.assertEqual(r.headers, {b'Local Filename': [local_fname], self.assertEqual(r.headers, {b'Local Filename': [fname_bytes],
b'Size': [b'17']}) b'Size': [b'17']})
self.assertTrue(os.path.exists(local_fname)) self.assertTrue(local_fname.exists())
with open(local_fname, "rb") as f: self.assertEqual(local_fname.read_bytes(), b"I have the power!")
self.assertEqual(f.read(), b"I have the power!") local_fname.unlink()
os.remove(local_fname)
return self._add_test_callbacks(d, _test) return self._add_test_callbacks(d, _test)
def _test_response_class(self, filename, response_class): def _test_response_class(self, filename, response_class):
f, local_fname = tempfile.mkstemp() f, local_fname = tempfile.mkstemp()
local_fname = to_bytes(local_fname) local_fname = Path(local_fname)
os.close(f) os.close(f)
meta = {} meta = {}
meta.update(self.req_meta) meta.update(self.req_meta)
@ -1107,7 +1105,7 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r): def _test(r):
self.assertEqual(type(r), response_class) self.assertEqual(type(r), response_class)
os.remove(local_fname) local_fname.unlink()
return self._add_test_callbacks(d, _test) return self._add_test_callbacks(d, _test)
def test_response_class_from_url(self): def test_response_class_from_url(self):
@ -1147,15 +1145,14 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
# setup dir and test file # setup dir and test file
self.directory = self.mktemp() self.directory = Path(self.mktemp())
os.mkdir(self.directory) self.directory.mkdir()
fp = FilePath(self.directory)
for filename, content in self.test_files: for filename, content in self.test_files:
fp.child(filename).setContent(content) (self.directory / filename).write_bytes(content)
# setup server for anonymous access # setup server for anonymous access
realm = FTPRealm(anonymousRoot=self.directory) realm = FTPRealm(anonymousRoot=str(self.directory))
p = portal.Portal(realm) p = portal.Portal(realm)
p.registerChecker(checkers.AllowAnonymousAccess(), p.registerChecker(checkers.AllowAnonymousAccess(),
credentials.IAnonymous) credentials.IAnonymous)

7
tests/test_downloadermiddleware_httpcompression.py
View File

@ -1,6 +1,6 @@
from gzip import GzipFile from gzip import GzipFile
from io import BytesIO from io import BytesIO
from os.path import join from pathlib import Path
from unittest import TestCase, SkipTest from unittest import TestCase, SkipTest
from warnings import catch_warnings from warnings import catch_warnings
@ -15,7 +15,7 @@ from tests import tests_datadir
from w3lib.encoding import resolve_encoding from w3lib.encoding import resolve_encoding
SAMPLEDIR = join(tests_datadir, 'compressed') SAMPLEDIR = Path(tests_datadir, 'compressed')
FORMAT = { FORMAT = {
'gzip': ('html-gzip.bin', 'gzip'), 'gzip': ('html-gzip.bin', 'gzip'),
@ -46,8 +46,7 @@ class HttpCompressionTest(TestCase):
samplefile, contentencoding = FORMAT[coding] samplefile, contentencoding = FORMAT[coding]
with open(join(SAMPLEDIR, samplefile), 'rb') as sample: body = (SAMPLEDIR / samplefile).read_bytes()
body = sample.read()
headers = { headers = {
'Server': 'Yaws/1.49 Yet Another Web Server', 'Server': 'Yaws/1.49 Yet Another Web Server',

4
tests/test_dupefilters.py
View File

@ -2,8 +2,8 @@ import hashlib
import tempfile import tempfile
import unittest import unittest
import shutil import shutil
import os
import sys import sys
from pathlib import Path
from testfixtures import LogCapture from testfixtures import LogCapture
from scrapy.dupefilters import RFPDupeFilter from scrapy.dupefilters import RFPDupeFilter
@ -157,7 +157,7 @@ class RFPDupeFilterTest(unittest.TestCase):
df.request_seen(r1) df.request_seen(r1)
df.close('finished') df.close('finished')
with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file: with Path(path, 'requests.seen').open('rb') as seen_file:
line = next(seen_file).decode() line = next(seen_file).decode()
assert not line.endswith('\r\r\n') assert not line.endswith('\r\r\n')
if sys.platform == 'win32': if sys.platform == 'win32':

6
tests/test_engine.py
View File

@ -10,11 +10,11 @@ module with the ``runserver`` argument::
python test_engine.py runserver python test_engine.py runserver
""" """
import os
import re import re
import subprocess import subprocess
import sys import sys
from collections import defaultdict from collections import defaultdict
from pathlib import Path
from threading import Timer from threading import Timer
from urllib.parse import urlparse from urllib.parse import urlparse
from dataclasses import dataclass from dataclasses import dataclass
@ -127,8 +127,8 @@ class ChangeCloseReasonSpider(TestSpider):
def start_test_site(debug=False): def start_test_site(debug=False):
root_dir = os.path.join(tests_datadir, "test_site") root_dir = Path(tests_datadir, "test_site")
r = static.File(root_dir) r = static.File(str(root_dir))
r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirect", util.Redirect(b"/redirected"))
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**18)] numbers = [str(x).encode("utf8") for x in range(2**18)]

155
tests/test_feedexport.py
View File

@ -3,7 +3,6 @@ import csv
import gzip import gzip
import json import json
import lzma import lzma
import os
import random import random
import shutil import shutil
import string import string
@ -15,6 +14,7 @@ from collections import defaultdict
from contextlib import ExitStack from contextlib import ExitStack
from io import BytesIO from io import BytesIO
from logging import getLogger from logging import getLogger
from os import PathLike
from pathlib import Path from pathlib import Path
from string import ascii_letters, digits from string import ascii_letters, digits
from unittest import mock from unittest import mock
@ -63,41 +63,41 @@ def printf_escape(string):
return string.replace('%', '%%') return string.replace('%', '%%')
def build_url(path): def build_url(path: str | PathLike[str]) -> str:
if path[0] != '/': path_str = str(path)
path = '/' + path if path_str[0] != '/':
return urljoin('file:', path) path_str = '/' + path_str
return urljoin('file:', path_str)
class FileFeedStorageTest(unittest.TestCase): class FileFeedStorageTest(unittest.TestCase):
def test_store_file_uri(self): def test_store_file_uri(self):
path = os.path.abspath(self.mktemp()) path = Path(self.mktemp()).resolve()
uri = path_to_file_uri(path) uri = path_to_file_uri(str(path))
return self._assert_stores(FileFeedStorage(uri), path) return self._assert_stores(FileFeedStorage(uri), path)
def test_store_file_uri_makedirs(self): def test_store_file_uri_makedirs(self):
path = os.path.abspath(self.mktemp()) path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt'
path = os.path.join(path, 'more', 'paths', 'file.txt') uri = path_to_file_uri(str(path))
uri = path_to_file_uri(path)
return self._assert_stores(FileFeedStorage(uri), path) return self._assert_stores(FileFeedStorage(uri), path)
def test_store_direct_path(self): def test_store_direct_path(self):
path = os.path.abspath(self.mktemp()) path = Path(self.mktemp()).resolve()
return self._assert_stores(FileFeedStorage(path), path) return self._assert_stores(FileFeedStorage(str(path)), path)
def test_store_direct_path_relative(self): def test_store_direct_path_relative(self):
path = self.mktemp() path = Path(self.mktemp())
return self._assert_stores(FileFeedStorage(path), path) return self._assert_stores(FileFeedStorage(str(path)), path)
def test_interface(self): def test_interface(self):
path = self.mktemp() path = self.mktemp()
st = FileFeedStorage(path) st = FileFeedStorage(path)
verifyObject(IFeedStorage, st) verifyObject(IFeedStorage, st)
def _store(self, feed_options=None): def _store(self, feed_options=None) -> Path:
path = os.path.abspath(self.mktemp()) path = Path(self.mktemp()).resolve()
storage = FileFeedStorage(path, feed_options=feed_options) storage = FileFeedStorage(str(path), feed_options=feed_options)
spider = scrapy.Spider("default") spider = scrapy.Spider("default")
file = storage.open(spider) file = storage.open(spider)
file.write(b"content") file.write(b"content")
@ -106,27 +106,26 @@ class FileFeedStorageTest(unittest.TestCase):
def test_append(self): def test_append(self):
path = self._store() path = self._store()
return self._assert_stores(FileFeedStorage(path), path, b"contentcontent") return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent")
def test_overwrite(self): def test_overwrite(self):
path = self._store({"overwrite": True}) path = self._store({"overwrite": True})
return self._assert_stores( return self._assert_stores(
FileFeedStorage(path, feed_options={"overwrite": True}), FileFeedStorage(str(path), feed_options={"overwrite": True}),
path path
) )
@defer.inlineCallbacks @defer.inlineCallbacks
def _assert_stores(self, storage, path, expected_content=b"content"): def _assert_stores(self, storage, path: Path, expected_content=b"content"):
spider = scrapy.Spider("default") spider = scrapy.Spider("default")
file = storage.open(spider) file = storage.open(spider)
file.write(b"content") file.write(b"content")
yield storage.store(file) yield storage.store(file)
self.assertTrue(os.path.exists(path)) self.assertTrue(path.exists())
try: try:
with open(path, 'rb') as fp: self.assertEqual(path.read_bytes(), expected_content)
self.assertEqual(fp.read(), expected_content)
finally: finally:
os.unlink(path) path.unlink()
class FTPFeedStorageTest(unittest.TestCase): class FTPFeedStorageTest(unittest.TestCase):
@ -152,13 +151,12 @@ class FTPFeedStorageTest(unittest.TestCase):
file.write(content) file.write(content)
return storage.store(file) return storage.store(file)
def _assert_stored(self, path, content): def _assert_stored(self, path: Path, content):
self.assertTrue(path.exists()) self.assertTrue(path.exists())
try: try:
with path.open('rb') as fp: self.assertEqual(path.read_bytes(), content)
self.assertEqual(fp.read(), content)
finally: finally:
os.unlink(str(path)) path.unlink()
@defer.inlineCallbacks @defer.inlineCallbacks
def test_append(self): def test_append(self):
@ -221,24 +219,24 @@ class BlockingFeedStorageTest(unittest.TestCase):
b = BlockingFeedStorage() b = BlockingFeedStorage()
tmp = b.open(self.get_test_spider()) tmp = b.open(self.get_test_spider())
tmp_path = os.path.dirname(tmp.name) tmp_path = Path(tmp.name).parent
self.assertEqual(tmp_path, tempfile.gettempdir()) self.assertEqual(str(tmp_path), tempfile.gettempdir())
def test_temp_file(self): def test_temp_file(self):
b = BlockingFeedStorage() b = BlockingFeedStorage()
tests_path = os.path.dirname(os.path.abspath(__file__)) tests_path = Path(__file__).resolve().parent
spider = self.get_test_spider({'FEED_TEMPDIR': tests_path}) spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)})
tmp = b.open(spider) tmp = b.open(spider)
tmp_path = os.path.dirname(tmp.name) tmp_path = Path(tmp.name).parent
self.assertEqual(tmp_path, tests_path) self.assertEqual(tmp_path, tests_path)
def test_invalid_folder(self): def test_invalid_folder(self):
b = BlockingFeedStorage() b = BlockingFeedStorage()
tests_path = os.path.dirname(os.path.abspath(__file__)) tests_path = Path(__file__).resolve().parent
invalid_path = os.path.join(tests_path, 'invalid_path') invalid_path = tests_path / 'invalid_path'
spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path}) spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)})
self.assertRaises(OSError, b.open, spider=spider) self.assertRaises(OSError, b.open, spider=spider)
@ -564,13 +562,13 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
class DummyBlockingFeedStorage(BlockingFeedStorage): class DummyBlockingFeedStorage(BlockingFeedStorage):
def __init__(self, uri, *args, feed_options=None): def __init__(self, uri, *args, feed_options=None):
self.path = file_uri_to_path(uri) self.path = Path(file_uri_to_path(uri))
def _store_in_thread(self, file): def _store_in_thread(self, file):
dirname = os.path.dirname(self.path) dirname = self.path.parent
if dirname and not os.path.exists(dirname): if dirname and not dirname.exists():
os.makedirs(dirname) dirname.mkdir(parents=True)
with open(self.path, 'ab') as output_file: with self.path.open('ab') as output_file:
output_file.write(file.read()) output_file.write(file.read())
file.close() file.close()
@ -613,10 +611,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):
foo = scrapy.Field() foo = scrapy.Field()
hello = scrapy.Field() hello = scrapy.Field()
def _random_temp_filename(self, inter_dir=''): def _random_temp_filename(self, inter_dir='') -> Path:
chars = [random.choice(ascii_letters + digits) for _ in range(15)] chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars) filename = ''.join(chars)
return os.path.join(self.temp_dir, inter_dir, filename) return Path(self.temp_dir, inter_dir, filename)
def setUp(self): def setUp(self):
self.temp_dir = tempfile.mkdtemp() self.temp_dir = tempfile.mkdtemp()
@ -702,18 +700,17 @@ class FeedExportTest(FeedExportTestBase):
yield crawler.crawl() yield crawler.crawl()
for file_path, feed_options in FEEDS.items(): for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)): if not Path(file_path).exists():
continue continue
with open(str(file_path), 'rb') as f: content[feed_options['format']] = Path(file_path).read_bytes()
content[feed_options['format']] = f.read()
finally: finally:
for file_path in FEEDS.keys(): for file_path in FEEDS.keys():
if not os.path.exists(str(file_path)): if not Path(file_path).exists():
continue continue
os.remove(str(file_path)) Path(file_path).unlink()
return content return content
@ -808,7 +805,7 @@ class FeedExportTest(FeedExportTestBase):
def test_stats_file_success(self): def test_stats_file_success(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
printf_escape(path_to_url(self._random_temp_filename())): { printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json", "format": "json",
} }
}, },
@ -823,7 +820,7 @@ class FeedExportTest(FeedExportTestBase):
def test_stats_file_failed(self): def test_stats_file_failed(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
printf_escape(path_to_url(self._random_temp_filename())): { printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json", "format": "json",
} }
}, },
@ -846,7 +843,7 @@ class FeedExportTest(FeedExportTestBase):
'AWS_ACCESS_KEY_ID': 'access_key', 'AWS_ACCESS_KEY_ID': 'access_key',
'AWS_SECRET_ACCESS_KEY': 'secret_key', 'AWS_SECRET_ACCESS_KEY': 'secret_key',
"FEEDS": { "FEEDS": {
printf_escape(path_to_url(self._random_temp_filename())): { printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json", "format": "json",
}, },
"s3://bucket/key/foo.csv": { "s3://bucket/key/foo.csv": {
@ -1427,12 +1424,11 @@ class FeedExportTest(FeedExportTestBase):
self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)
@defer.inlineCallbacks @defer.inlineCallbacks
def test_pathlib_uri(self): def test_str_uri(self):
feed_path = Path(self._random_temp_filename())
settings = { settings = {
'FEED_STORE_EMPTY': True, 'FEED_STORE_EMPTY': True,
'FEEDS': { 'FEEDS': {
feed_path: {'format': 'csv'} str(self._random_temp_filename()): {'format': 'csv'}
}, },
} }
data = yield self.exported_no_data(settings) data = yield self.exported_no_data(settings)
@ -1538,8 +1534,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
def close(self): def close(self):
self.file.close() self.file.close()
def _named_tempfile(self, name): def _named_tempfile(self, name) -> str:
return os.path.join(self.temp_dir, name) return str(Path(self.temp_dir, name))
@defer.inlineCallbacks @defer.inlineCallbacks
def run_and_export(self, spider_cls, settings): def run_and_export(self, spider_cls, settings):
@ -1559,18 +1555,17 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
yield crawler.crawl() yield crawler.crawl()
for file_path, feed_options in FEEDS.items(): for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)): if not Path(file_path).exists():
continue continue
with open(str(file_path), 'rb') as f: content[str(file_path)] = Path(file_path).read_bytes()
content[str(file_path)] = f.read()
finally: finally:
for file_path in FEEDS.keys(): for file_path in FEEDS.keys():
if not os.path.exists(str(file_path)): if not Path(file_path).exists():
continue continue
os.remove(str(file_path)) Path(file_path).unlink()
return content return content
@ -2031,11 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
yield crawler.crawl() yield crawler.crawl()
for path, feed in FEEDS.items(): for path, feed in FEEDS.items():
dir_name = os.path.dirname(path) dir_name = Path(path).parent
for file in sorted(os.listdir(dir_name)): for file in sorted(dir_name.iterdir()):
with open(os.path.join(dir_name, file), 'rb') as f: content[feed['format']].append(file.read_bytes())
data = f.read()
content[feed['format']].append(data)
finally: finally:
self.tearDown() self.tearDown()
defer.returnValue(content) defer.returnValue(content)
@ -2045,7 +2038,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {} settings = settings or {}
settings.update({ settings.update({
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'},
}, },
}) })
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2061,7 +2054,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {} settings = settings or {}
settings.update({ settings.update({
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'},
}, },
}) })
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2077,7 +2070,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {} settings = settings or {}
settings.update({ settings.update({
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
}, },
}) })
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2094,8 +2087,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {} settings = settings or {}
settings.update({ settings.update({
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'},
}, },
}) })
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2120,7 +2113,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {} settings = settings or {}
settings.update({ settings.update({
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'},
}, },
}) })
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2137,7 +2130,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {} settings = settings or {}
settings.update({ settings.update({
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'},
}, },
}) })
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2184,7 +2177,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
for fmt in ('json', 'jsonlines', 'xml', 'csv'): for fmt in ('json', 'jsonlines', 'xml', 'csv'):
settings = { settings = {
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
}, },
'FEED_EXPORT_BATCH_ITEM_COUNT': 1 'FEED_EXPORT_BATCH_ITEM_COUNT': 1
} }
@ -2204,7 +2197,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
for fmt, expctd in formats: for fmt, expctd in formats:
settings = { settings = {
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
}, },
'FEED_STORE_EMPTY': True, 'FEED_STORE_EMPTY': True,
'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_INDENT': None,
@ -2237,19 +2230,19 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = { settings = {
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'json', self._file_mark): { self._random_temp_filename() / 'json' / self._file_mark: {
'format': 'json', 'format': 'json',
'indent': 0, 'indent': 0,
'fields': ['bar'], 'fields': ['bar'],
'encoding': 'utf-8', 'encoding': 'utf-8',
}, },
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): { self._random_temp_filename() / 'xml' / self._file_mark: {
'format': 'xml', 'format': 'xml',
'indent': 2, 'indent': 2,
'fields': ['foo'], 'fields': ['foo'],
'encoding': 'latin-1', 'encoding': 'latin-1',
}, },
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { self._random_temp_filename() / 'csv' / self._file_mark: {
'format': 'csv', 'format': 'csv',
'indent': None, 'indent': None,
'fields': ['foo', 'bar'], 'fields': ['foo', 'bar'],
@ -2272,7 +2265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
} }
settings = { settings = {
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), 'json', self._file_mark): { self._random_temp_filename() / 'json' / self._file_mark: {
'format': 'json', 'format': 'json',
'indent': None, 'indent': None,
'encoding': 'utf-8', 'encoding': 'utf-8',
@ -2299,7 +2292,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
] ]
settings = { settings = {
'FEEDS': { 'FEEDS': {
os.path.join(self._random_temp_filename(), '%(batch_time)s'): { self._random_temp_filename() / '%(batch_time)s': {
'format': 'json', 'format': 'json',
}, },
}, },
@ -2312,7 +2305,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
def test_stats_batch_file_success(self): def test_stats_batch_file_success(self):
settings = { settings = {
"FEEDS": { "FEEDS": {
build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): { build_url(str(self._random_temp_filename() / "json" / self._file_mark)): {
"format": "json", "format": "json",
} }
}, },

15
tests/test_http2_client_protocol.py
View File

@ -1,10 +1,10 @@
import json import json
import os
import random import random
import re import re
import shutil import shutil
import string import string
from ipaddress import IPv4Address from ipaddress import IPv4Address
from pathlib import Path
from unittest import mock, skipIf from unittest import mock, skipIf
from urllib.parse import urlencode from urllib.parse import urlencode
@ -163,9 +163,8 @@ class RequestHeaders(LeafResource):
return bytes(json.dumps(headers), 'utf-8') return bytes(json.dumps(headers), 'utf-8')
def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate:
with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = key_file.read_text() + certificate_file.read_text()
pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
return PrivateCertificate.loadPEM(pem) return PrivateCertificate.loadPEM(pem)
@ -173,12 +172,12 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2ClientProtocolTestCase(TestCase): class Https2ClientProtocolTestCase(TestCase):
scheme = 'https' scheme = 'https'
key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') key_file = Path(__file__).parent / 'keys' / 'localhost.key'
certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt') certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt'
def _init_resource(self): def _init_resource(self):
self.temp_directory = self.mktemp() self.temp_directory = self.mktemp()
os.mkdir(self.temp_directory) Path(self.temp_directory).mkdir()
r = File(self.temp_directory) r = File(self.temp_directory)
r.putChild(b'get-data-html-small', GetDataHtmlSmall()) r.putChild(b'get-data-html-small', GetDataHtmlSmall())
r.putChild(b'get-data-html-large', GetDataHtmlLarge()) r.putChild(b'get-data-html-large', GetDataHtmlLarge())
@ -202,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase):
# Start server for testing # Start server for testing
self.hostname = 'localhost' self.hostname = 'localhost'
context_factory = ssl_context_factory(self.key_file, self.certificate_file) context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file))
server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
self.server = yield server_endpoint.listen(self.site) self.server = yield server_endpoint.listen(self.site)

14
tests/test_pipeline_crawl.py
View File

@ -1,5 +1,5 @@
import os
import shutil import shutil
from pathlib import Path
from testfixtures import LogCapture from testfixtures import LogCapture
from twisted.internet import defer from twisted.internet import defer
@ -61,12 +61,12 @@ class FileDownloadCrawlTestCase(TestCase):
self.mockserver.__enter__() self.mockserver.__enter__()
# prepare a directory for storing files # prepare a directory for storing files
self.tmpmediastore = self.mktemp() self.tmpmediastore = Path(self.mktemp())
os.mkdir(self.tmpmediastore) self.tmpmediastore.mkdir()
self.settings = { self.settings = {
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'ITEM_PIPELINES': {self.pipeline_class: 1}, 'ITEM_PIPELINES': {self.pipeline_class: 1},
self.store_setting_key: self.tmpmediastore, self.store_setting_key: str(self.tmpmediastore),
} }
self.runner = CrawlerRunner(self.settings) self.runner = CrawlerRunner(self.settings)
self.items = [] self.items = []
@ -111,9 +111,7 @@ class FileDownloadCrawlTestCase(TestCase):
# check that the image files where actually written to the media store # check that the image files where actually written to the media store
for item in items: for item in items:
for i in item[self.media_key]: for i in item[self.media_key]:
self.assertTrue( self.assertTrue((self.tmpmediastore / i['path']).exists())
os.path.exists(
os.path.join(self.tmpmediastore, i['path'])))
def _assert_files_download_failure(self, crawler, items, code, logs): def _assert_files_download_failure(self, crawler, items, code, logs):
@ -133,7 +131,7 @@ class FileDownloadCrawlTestCase(TestCase):
self.assertEqual(logs.count(file_dl_failure), 3) self.assertEqual(logs.count(file_dl_failure), 3)
# check that no files were written to the media store # check that no files were written to the media store
self.assertEqual(os.listdir(self.tmpmediastore), []) self.assertEqual([x for x in self.tmpmediastore.iterdir()], [])
@defer.inlineCallbacks @defer.inlineCallbacks
def test_download_media(self): def test_download_media(self):

3
tests/test_pipeline_files.py
View File

@ -3,6 +3,7 @@ import random
import time import time
from datetime import datetime from datetime import datetime
from io import BytesIO from io import BytesIO
from pathlib import Path
from shutil import rmtree from shutil import rmtree
from tempfile import mkdtemp from tempfile import mkdtemp
from unittest import mock from unittest import mock
@ -89,7 +90,7 @@ class FilesPipelineTestCase(unittest.TestCase):
self.assertEqual(self.pipeline.store.basedir, self.tempdir) self.assertEqual(self.pipeline.store.basedir, self.tempdir)
path = 'some/image/key.jpg' path = 'some/image/key.jpg'
fullpath = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg')
self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath) self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath)
@defer.inlineCallbacks @defer.inlineCallbacks

6
tests/test_proxy_connect.py
View File

@ -2,6 +2,7 @@ import json
import os import os
import re import re
import sys import sys
from pathlib import Path
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
from urllib.parse import urlsplit, urlunsplit from urllib.parse import urlsplit, urlunsplit
from testfixtures import LogCapture from testfixtures import LogCapture
@ -27,14 +28,13 @@ from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump" sys.argv[0] = "mitmdump"
sys.exit(mitmdump()) sys.exit(mitmdump())
""" """
cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem'
'keys', 'mitmproxy-ca.pem')
self.proc = Popen([sys.executable, self.proc = Popen([sys.executable,
'-c', script, '-c', script,
'--listen-host', '127.0.0.1', '--listen-host', '127.0.0.1',
'--listen-port', '0', '--listen-port', '0',
'--proxyauth', f'{self.auth_user}:{self.auth_pass}', '--proxyauth', f'{self.auth_user}:{self.auth_pass}',
'--certs', cert_path, '--certs', str(cert_path),
'--ssl-insecure', '--ssl-insecure',
], ],
stdout=PIPE, env=get_testenv()) stdout=PIPE, env=get_testenv())

40
tests/test_spiderloader/__init__.py
View File

@ -1,7 +1,7 @@
import sys import sys
import os
import shutil import shutil
import warnings import warnings
from pathlib import Path
from zope.interface.verify import verifyObject from zope.interface.verify import verifyObject
from twisted.trial import unittest from twisted.trial import unittest
@ -17,10 +17,10 @@ from scrapy.settings import Settings
from scrapy.http import Request from scrapy.http import Request
from scrapy.crawler import CrawlerRunner from scrapy.crawler import CrawlerRunner
module_dir = os.path.dirname(os.path.abspath(__file__)) module_dir = Path(__file__).resolve().parent
def _copytree(source, target): def _copytree(source: Path, target: Path):
try: try:
shutil.copytree(source, target) shutil.copytree(source, target)
except shutil.Error: except shutil.Error:
@ -30,18 +30,18 @@ def _copytree(source, target):
class SpiderLoaderTest(unittest.TestCase): class SpiderLoaderTest(unittest.TestCase):
def setUp(self): def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders') orig_spiders_dir = module_dir / 'test_spiders'
self.tmpdir = tempfile.mkdtemp() self.tmpdir = Path(tempfile.mkdtemp())
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
_copytree(orig_spiders_dir, self.spiders_dir) _copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir) sys.path.append(str(self.tmpdir))
settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
self.spider_loader = SpiderLoader.from_settings(settings) self.spider_loader = SpiderLoader.from_settings(settings)
def tearDown(self): def tearDown(self):
del self.spider_loader del self.spider_loader
del sys.modules['test_spiders_xxx'] del sys.modules['test_spiders_xxx']
sys.path.remove(self.tmpdir) sys.path.remove(str(self.tmpdir))
def test_interface(self): def test_interface(self):
verifyObject(ISpiderLoader, self.spider_loader) verifyObject(ISpiderLoader, self.spider_loader)
@ -135,22 +135,22 @@ class SpiderLoaderTest(unittest.TestCase):
class DuplicateSpiderNameLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def setUp(self): def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders') orig_spiders_dir = module_dir / 'test_spiders'
self.tmpdir = self.mktemp() self.tmpdir = Path(self.mktemp())
os.mkdir(self.tmpdir) self.tmpdir.mkdir()
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
_copytree(orig_spiders_dir, self.spiders_dir) _copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir) sys.path.append(str(self.tmpdir))
self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
def tearDown(self): def tearDown(self):
del sys.modules['test_spiders_xxx'] del sys.modules['test_spiders_xxx']
sys.path.remove(self.tmpdir) sys.path.remove(str(self.tmpdir))
def test_dupename_warning(self): def test_dupename_warning(self):
# copy 1 spider module so as to have duplicate spider name # copy 1 spider module so as to have duplicate spider name
shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'), shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py',
os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py')) self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py')
with warnings.catch_warnings(record=True) as w: with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings) spider_loader = SpiderLoader.from_settings(self.settings)
@ -171,10 +171,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def test_multiple_dupename_warning(self): def test_multiple_dupename_warning(self):
# copy 2 spider modules so as to have duplicate spider name # copy 2 spider modules so as to have duplicate spider name
# This should issue 2 warning, 1 for each duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name
shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'), shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py',
os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py')) self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py')
shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'), shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py',
os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py')) self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py')
with warnings.catch_warnings(record=True) as w: with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings) spider_loader = SpiderLoader.from_settings(self.settings)

4
tests/test_spiderstate.py
View File

@ -1,5 +1,5 @@
import os
from datetime import datetime from datetime import datetime
from pathlib import Path
import shutil import shutil
from twisted.trial import unittest from twisted.trial import unittest
@ -13,7 +13,7 @@ class SpiderStateTest(unittest.TestCase):
def test_store_load(self): def test_store_load(self):
jobdir = self.mktemp() jobdir = self.mktemp()
os.mkdir(jobdir) Path(jobdir).mkdir()
try: try:
spider = Spider(name='default') spider = Spider(name='default')
dt = datetime.now() dt = datetime.now()

44
tests/test_utils_gz.py
View File

@ -1,5 +1,5 @@
import unittest import unittest
from os.path import join from pathlib import Path
from w3lib.encoding import html_to_unicode from w3lib.encoding import html_to_unicode
@ -8,46 +8,40 @@ from scrapy.http import Response
from tests import tests_datadir from tests import tests_datadir
SAMPLEDIR = join(tests_datadir, 'compressed') SAMPLEDIR = Path(tests_datadir, 'compressed')
class GunzipTest(unittest.TestCase): class GunzipTest(unittest.TestCase):
def test_gunzip_basic(self): def test_gunzip_basic(self):
with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes())
r1 = Response("http://www.example.com", body=f.read()) self.assertTrue(gzip_magic_number(r1))
self.assertTrue(gzip_magic_number(r1))
r2 = Response("http://www.example.com", body=gunzip(r1.body)) r2 = Response("http://www.example.com", body=gunzip(r1.body))
self.assertFalse(gzip_magic_number(r2)) self.assertFalse(gzip_magic_number(r2))
self.assertEqual(len(r2.body), 9950) self.assertEqual(len(r2.body), 9950)
def test_gunzip_truncated(self): def test_gunzip_truncated(self):
with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: text = gunzip((SAMPLEDIR / 'truncated-crc-error.gz').read_bytes())
text = gunzip(f.read()) assert text.endswith(b'</html')
assert text.endswith(b'</html')
def test_gunzip_no_gzip_file_raises(self): def test_gunzip_no_gzip_file_raises(self):
with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f: self.assertRaises(IOError, gunzip, (SAMPLEDIR / 'feed-sample1.xml').read_bytes())
self.assertRaises(IOError, gunzip, f.read())
def test_gunzip_truncated_short(self): def test_gunzip_truncated_short(self):
with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'truncated-crc-error-short.gz').read_bytes())
r1 = Response("http://www.example.com", body=f.read()) self.assertTrue(gzip_magic_number(r1))
self.assertTrue(gzip_magic_number(r1))
r2 = Response("http://www.example.com", body=gunzip(r1.body)) r2 = Response("http://www.example.com", body=gunzip(r1.body))
assert r2.body.endswith(b'</html>') assert r2.body.endswith(b'</html>')
self.assertFalse(gzip_magic_number(r2)) self.assertFalse(gzip_magic_number(r2))
def test_is_gzipped_empty(self): def test_is_gzipped_empty(self):
r1 = Response("http://www.example.com") r1 = Response("http://www.example.com")
self.assertFalse(gzip_magic_number(r1)) self.assertFalse(gzip_magic_number(r1))
def test_gunzip_illegal_eof(self): def test_gunzip_illegal_eof(self):
with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1]
text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1] expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8")
with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o: self.assertEqual(len(text), len(expected_text))
expected_text = o.read().decode("utf-8") self.assertEqual(text, expected_text)
self.assertEqual(len(text), len(expected_text))
self.assertEqual(text, expected_text)

7
tests/test_utils_iterators.py
View File

@ -1,5 +1,3 @@
import os
from pytest import mark from pytest import mark
from twisted.trial import unittest from twisted.trial import unittest
@ -303,11 +301,6 @@ class LxmlXmliterTestCase(XmliterTestCase):
class UtilsCsvTestCase(unittest.TestCase): class UtilsCsvTestCase(unittest.TestCase):
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')
sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv')
sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv')
def test_csviter_defaults(self): def test_csviter_defaults(self):
body = get_testdata('feeds', 'feed-sample3.csv') body = get_testdata('feeds', 'feed-sample3.csv')
response = TextResponse(url="http://example.com/", body=body) response = TextResponse(url="http://example.com/", body=body)

3
tests/test_utils_misc/__init__.py
View File

@ -1,6 +1,7 @@
import sys import sys
import os import os
import unittest import unittest
from pathlib import Path
from unittest import mock from unittest import mock
from scrapy.item import Item, Field from scrapy.item import Item, Field
@ -55,7 +56,7 @@ class UtilsMiscTestCase(unittest.TestCase):
self.assertRaises(ImportError, walk_modules, 'nomodule999') self.assertRaises(ImportError, walk_modules, 'nomodule999')
def test_walk_modules_egg(self): def test_walk_modules_egg(self):
egg = os.path.join(os.path.dirname(__file__), 'test.egg') egg = str(Path(__file__).parent / 'test.egg')
sys.path.append(egg) sys.path.append(egg)
try: try:
mods = walk_modules('testegg') mods = walk_modules('testegg')

17
tests/test_utils_project.py
View File

@ -4,6 +4,7 @@ import tempfile
import shutil import shutil
import contextlib import contextlib
import warnings import warnings
from pathlib import Path
from pytest import warns from pytest import warns
@ -18,9 +19,7 @@ def inside_a_project():
try: try:
os.chdir(project_dir) os.chdir(project_dir)
with open('scrapy.cfg', 'w') as f: Path('scrapy.cfg').touch()
# create an empty scrapy.cfg
f.close()
yield project_dir yield project_dir
finally: finally:
@ -31,20 +30,20 @@ def inside_a_project():
class ProjectUtilsTest(unittest.TestCase): class ProjectUtilsTest(unittest.TestCase):
def test_data_path_outside_project(self): def test_data_path_outside_project(self):
self.assertEqual( self.assertEqual(
os.path.join('.scrapy', 'somepath'), str(Path('.scrapy', 'somepath')),
data_path('somepath') data_path('somepath')
) )
abspath = os.path.join(os.path.sep, 'absolute', 'path') abspath = str(Path(os.path.sep, 'absolute', 'path'))
self.assertEqual(abspath, data_path(abspath)) self.assertEqual(abspath, data_path(abspath))
def test_data_path_inside_project(self): def test_data_path_inside_project(self):
with inside_a_project() as proj_path: with inside_a_project() as proj_path:
expected = os.path.join(proj_path, '.scrapy', 'somepath') expected = Path(proj_path, '.scrapy', 'somepath')
self.assertEqual( self.assertEqual(
os.path.realpath(expected), expected.resolve(),
os.path.realpath(data_path('somepath')) Path(data_path('somepath')).resolve()
) )
abspath = os.path.join(os.path.sep, 'absolute', 'path') abspath = str(Path(os.path.sep, 'absolute', 'path').resolve())
self.assertEqual(abspath, data_path(abspath)) self.assertEqual(abspath, data_path(abspath))

12
tests/test_utils_response.py
View File

@ -1,6 +1,6 @@
import os
import unittest import unittest
import warnings import warnings
from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exceptions import ScrapyDeprecationWarning
@ -39,10 +39,9 @@ class ResponseUtilsTest(unittest.TestCase):
def browser_open(burl): def browser_open(burl):
path = urlparse(burl).path path = urlparse(burl).path
if not os.path.exists(path): if not path or not Path(path).exists():
path = burl.replace('file://', '') path = burl.replace('file://', '')
with open(path, "rb") as f: bbody = Path(path).read_bytes()
bbody = f.read()
self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody) self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
return True return True
response = HtmlResponse(url, body=body) response = HtmlResponse(url, body=body)
@ -98,10 +97,9 @@ class ResponseUtilsTest(unittest.TestCase):
def check_base_url(burl): def check_base_url(burl):
path = urlparse(burl).path path = urlparse(burl).path
if not os.path.exists(path): if not path or not Path(path).exists():
path = burl.replace('file://', '') path = burl.replace('file://', '')
with open(path, "rb") as f: bbody = Path(path).read_bytes()
bbody = f.read()
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1) self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
return True return True

22
tests/test_utils_template.py
View File

@ -1,4 +1,4 @@
import os from pathlib import Path
from shutil import rmtree from shutil import rmtree
from tempfile import mkdtemp from tempfile import mkdtemp
import unittest import unittest
@ -22,21 +22,19 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
template = 'from ${project_name}.spiders.${name} import ${classname}' template = 'from ${project_name}.spiders.${name} import ${classname}'
rendered = 'from proj.spiders.spi import TheSpider' rendered = 'from proj.spiders.spi import TheSpider'
template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') template_path = Path(self.tmp_path, 'templ.py.tmpl')
render_path = os.path.join(self.tmp_path, 'templ.py') render_path = Path(self.tmp_path, 'templ.py')
with open(template_path, 'wb') as tmpl_file: template_path.write_text(template, encoding='utf8')
tmpl_file.write(template.encode('utf8')) assert template_path.is_file() # Failure of test itself
assert os.path.isfile(template_path) # Failure of test itself
render_templatefile(template_path, **context) render_templatefile(str(template_path), **context)
self.assertFalse(os.path.exists(template_path)) self.assertFalse(template_path.exists())
with open(render_path, 'rb') as result: self.assertEqual(render_path.read_text(encoding='utf8'), rendered)
self.assertEqual(result.read().decode('utf8'), rendered)
os.remove(render_path) render_path.unlink()
assert not os.path.exists(render_path) # Failure of test itself assert not render_path.exists() # Failure of test itself
if '__main__' == __name__: if '__main__' == __name__:

19
tests/test_webclient.py
View File

@ -2,8 +2,8 @@
from twisted.internet import defer from twisted.internet import defer
Tests borrowed from the twisted.web.client tests. Tests borrowed from the twisted.web.client tests.
""" """
import os
import shutil import shutil
from pathlib import Path
import OpenSSL.SSL import OpenSSL.SSL
from twisted.trial import unittest from twisted.trial import unittest
@ -15,7 +15,6 @@ except ImportError:
# deprecated in Twisted 19.7.0 # deprecated in Twisted 19.7.0
# (remove once we bump our requirement past that version) # (remove once we bump our requirement past that version)
from twisted.test.proto_helpers import StringTransport from twisted.test.proto_helpers import StringTransport
from twisted.python.filepath import FilePath
from twisted.protocols.policies import WrappingFactory from twisted.protocols.policies import WrappingFactory
from twisted.internet.defer import inlineCallbacks from twisted.internet.defer import inlineCallbacks
@ -230,10 +229,10 @@ class WebClientTestCase(unittest.TestCase):
return reactor.listenTCP(0, site, interface="127.0.0.1") return reactor.listenTCP(0, site, interface="127.0.0.1")
def setUp(self): def setUp(self):
self.tmpname = self.mktemp() self.tmpname = Path(self.mktemp())
os.mkdir(self.tmpname) self.tmpname.mkdir()
FilePath(self.tmpname).child("file").setContent(b"0123456789") (self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(self.tmpname) r = static.File(str(self.tmpname))
r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"error", ErrorResource()) r.putChild(b"error", ErrorResource())
@ -379,10 +378,10 @@ class WebClientSSLTestCase(unittest.TestCase):
return f"https://127.0.0.1:{self.portno}/{path}" return f"https://127.0.0.1:{self.portno}/{path}"
def setUp(self): def setUp(self):
self.tmpname = self.mktemp() self.tmpname = Path(self.mktemp())
os.mkdir(self.tmpname) self.tmpname.mkdir()
FilePath(self.tmpname).child("file").setContent(b"0123456789") (self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(self.tmpname) r = static.File(str(self.tmpname))
r.putChild(b"payload", PayloadResource()) r.putChild(b"payload", PayloadResource())
self.site = server.Site(r, timeout=None) self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site) self.wrapper = WrappingFactory(self.site)