RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use pathlib

This commit is contained in:
Matthew Donoughe 2022-10-17 17:40:10 -04:00
parent 1c7f3ebd75
commit 40d9ca3bdd
No known key found for this signature in database
GPG Key ID: 838812402CA8C19D
60 changed files with 595 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conftest.py
View File

@ -21,7 +21,7 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
with open('tests/ignores.txt') as reader:
with Path('tests/ignores.txt').open() as reader:
for line in reader:
file_path = line.strip()
if file_path and file_path[0] != '#':

4
docs/Makefile
View File

@ -86,8 +86,8 @@ coverage: BUILDER = coverage
coverage: build
htmlview: html
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
os.path.realpath('build/html/index.html'))"
$(PYTHON) -c "import webbrowser; from pathlib import Path; \
webbrowser.open('file://' + Path('build/html/index.html').resolve())"
clean:
-rm -rf build/*

9
docs/conf.py
View File

@ -11,13 +11,12 @@
import sys
from datetime import datetime
from os import path
from pathlib import Path
# If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
sys.path.append(path.join(path.dirname(__file__), "_ext"))
sys.path.insert(0, path.dirname(path.dirname(__file__)))
# is relative to the documentation root, use Path.absolute to make it absolute.
sys.path.append(str(Path(__file__).parent / "_ext"))
sys.path.insert(0, str(Path(__file__).parent.parent))
# General configuration

9
docs/conftest.py
View File

@ -1,5 +1,5 @@
import os
from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
from pathlib import Path
from scrapy.http.response.html import HtmlResponse
from sybil import Sybil
@ -12,10 +12,9 @@ from sybil.parsers.doctest import DocTestParser
from sybil.parsers.skip import skip
def load_response(url, filename):
input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
with open(input_path, 'rb') as input_file:
return HtmlResponse(url, body=input_file.read())
def load_response(url: str, filename: str) -> HtmlResponse:
input_path = Path(__file__).parent / '_tests' / filename
return HtmlResponse(url, body=input_path.read_bytes())
def setup(namespace):

10
docs/intro/tutorial.rst
View File

@ -85,6 +85,8 @@ page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
from pathlib import Path
import scrapy
@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file named
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
self.log(f'Saved file {filename}')
@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementation
of :meth:`~scrapy.Spider.start_requests` to create the initial requests
for your spider::
from pathlib import Path
import scrapy
@ -191,8 +194,7 @@ for your spider::
def parse(self, response):
page = response.url.split("/")[-2]
filename = f'quotes-{page}.html'
with open(filename, 'wb') as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
The :meth:`~scrapy.Spider.parse` method will be called to handle each
of the requests for those URLs, even though we haven't explicitly told Scrapy

4
docs/topics/item-pipeline.rst
View File

@ -186,6 +186,7 @@ item.
::
import hashlib
from pathlib import Path
from urllib.parse import quote
import scrapy
@ -214,8 +215,7 @@ item.
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = f"{url_hash}.png"
with open(filename, "wb") as f:
f.write(response.body)
Path(filename).write_bytes(response.body)
# Store filename in item.
adapter["screenshot_filename"] = filename

9
docs/topics/media-pipeline.rst
View File

@ -156,7 +156,6 @@ By overriding ``file_path`` like this:
.. code-block:: python
import hashlib
from os.path import splitext
def file_path(self, request, response=None, info=None, *, item=None):
image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5)
@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.files import FilesPipeline
@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.
@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline:
approach to download all files into the ``files`` folder with their
original filenames (e.g. ``files/foo.png``)::
import os
from pathlib import PurePosixPath
from urllib.parse import urlparse
from scrapy.pipelines.images import ImagesPipeline
@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
return 'files/' + PurePosixPath(urlparse(request.url).path).name
Similarly, you can use the ``item`` to determine the file path based on some item
property.

9
docs/utils/linkfix.py
View File

@ -13,6 +13,7 @@ Author: dufferzafar
"""
import re
from pathlib import Path
def main():
@ -27,7 +28,7 @@ def main():
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
with Path("build/linkcheck/output.txt").open() as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
@ -51,14 +52,12 @@ def main():
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
Path(_filename).write_text(_contents)
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = Path(_filename).read_text()
_contents = _contents.replace(match.group(3), match.group(4))
else:

4
scrapy/commands/__init__.py
View File

@ -3,6 +3,7 @@ Base class for Scrapy commands
"""
import os
import argparse
from pathlib import Path
from typing import Any, Dict
from twisted.python import failure
@ -93,8 +94,7 @@ class ScrapyCommand:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep)
if opts.pdb:
failure.startDebugMode()

48
scrapy/commands/genspider.py
View File

@ -2,8 +2,9 @@ import os
import shutil
import string
from pathlib import Path
from importlib import import_module
from os.path import join, dirname, abspath, exists, splitext
from typing import Optional
from urllib.parse import urlparse
import scrapy
@ -62,8 +63,7 @@ class Command(ScrapyCommand):
if opts.dump:
template_file = self._find_template(opts.dump)
if template_file:
with open(template_file, "r") as f:
print(f.read())
print(template_file.read_text())
return
if len(args) != 2:
raise UsageError()
@ -98,11 +98,11 @@ class Command(ScrapyCommand):
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = abspath(dirname(spiders_module.__file__))
spiders_dir = Path(spiders_module.__file__).parent.resolve()
else:
spiders_module = None
spiders_dir = "."
spider_file = f"{join(spiders_dir, module)}.py"
spiders_dir = Path(".")
spider_file = f"{spiders_dir / module}.py"
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print(f"Created spider {name!r} using template {template_name!r} ",
@ -110,24 +110,25 @@ class Command(ScrapyCommand):
if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}")
def _find_template(self, template):
template_file = join(self.templates_dir, f'{template}.tmpl')
if exists(template_file):
def _find_template(self, template: str) -> Optional[Path]:
template_file = Path(self.templates_dir, f'{template}.tmpl')
if template_file.exists():
return template_file
print(f"Unable to find template: {template}\n")
print('Use "scrapy genspider --list" to see all available templates.')
def _list_templates(self):
print("Available templates:")
for filename in sorted(os.listdir(self.templates_dir)):
if filename.endswith('.tmpl'):
print(f" {splitext(filename)[0]}")
for file in sorted(Path(self.templates_dir).iterdir()):
if file.suffix == '.tmpl':
print(f" {file.stem}")
def _spider_exists(self, name):
def _spider_exists(self, name: str) -> bool:
if not self.settings.get('NEWSPIDER_MODULE'):
# if run as a standalone command and file with same filename already exists
if exists(name + ".py"):
print(f"{abspath(name + '.py')} already exists")
path = Path(name + ".py")
if path.exists():
print(f"{path.resolve()} already exists")
return True
return False
@ -143,17 +144,18 @@ class Command(ScrapyCommand):
# a file with the same name exists in the target directory
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
spiders_dir = dirname(spiders_module.__file__)
spiders_dir_abs = abspath(spiders_dir)
if exists(join(spiders_dir_abs, name + ".py")):
print(f"{join(spiders_dir_abs, (name + '.py'))} already exists")
spiders_dir = Path(spiders_module.__file__).parent
spiders_dir_abs = spiders_dir.resolve()
path = spiders_dir_abs / (name + ".py")
if path.exists():
print(f"{path} already exists")
return True
return False
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
def templates_dir(self) -> str:
return str(Path(
self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'spiders'
)
))

21
scrapy/commands/runspider.py
View File

@ -1,22 +1,23 @@
import sys
import os
from os import PathLike
from pathlib import Path
from importlib import import_module
from types import ModuleType
from scrapy.utils.spider import iter_spider_classes
from scrapy.exceptions import UsageError
from scrapy.commands import BaseRunSpiderCommand
def _import_file(filepath):
abspath = os.path.abspath(filepath)
dirname, file = os.path.split(abspath)
fname, fext = os.path.splitext(file)
if fext not in ('.py', '.pyw'):
def _import_file(filepath: str | PathLike[str]) -> ModuleType:
abspath = Path(filepath).resolve()
dirname = str(abspath.parent)
if abspath.suffix not in ('.py', '.pyw'):
raise ValueError(f"Not a Python source file: {abspath}")
if dirname:
sys.path = [dirname] + sys.path
try:
module = import_module(fname)
module = import_module(abspath.stem)
finally:
if dirname:
sys.path.pop(0)
@ -40,13 +41,13 @@ class Command(BaseRunSpiderCommand):
def run(self, args, opts):
if len(args) != 1:
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
filename = Path(args[0])
if not filename.exists():
raise UsageError(f"File not found: {filename}\n")
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError(f"Unable to load {filename!r}: {e}\n")
raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError(f"No spider found in file: {filename}\n")

44
scrapy/commands/startproject.py
View File

@ -2,7 +2,7 @@ import re
import os
import string
from importlib.util import find_spec
from os.path import join, exists, abspath
from pathlib import Path
from shutil import ignore_patterns, move, copy2, copystat
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
@ -54,7 +54,7 @@ class Command(ScrapyCommand):
return True
return False
def _copytree(self, src, dst):
def _copytree(self, src: Path, dst: Path):
"""
Since the original function always creates the directory, to resolve
the issue a new function had to be created. It's a simple copy and
@ -64,19 +64,19 @@ class Command(ScrapyCommand):
https://github.com/scrapy/scrapy/pull/2005
"""
ignore = IGNORE
names = os.listdir(src)
names = [x.name for x in src.iterdir()]
ignored_names = ignore(src, names)
if not os.path.exists(dst):
os.makedirs(dst)
if not dst.exists():
dst.mkdir(parents=True)
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
if os.path.isdir(srcname):
srcname = src / name
dstname = dst / name
if srcname.is_dir():
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
@ -90,36 +90,36 @@ class Command(ScrapyCommand):
raise UsageError()
project_name = args[0]
project_dir = args[0]
if len(args) == 2:
project_dir = args[1]
project_dir = Path(args[1])
else:
project_dir = Path(args[0])
if exists(join(project_dir, 'scrapy.cfg')):
if (project_dir / 'scrapy.cfg').exists():
self.exitcode = 1
print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}')
print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}')
return
if not self._is_valid_name(project_name):
self.exitcode = 1
return
self._copytree(self.templates_dir, abspath(project_dir))
move(join(project_dir, 'module'), join(project_dir, project_name))
self._copytree(Path(self.templates_dir), project_dir.resolve())
move(project_dir / 'module', project_dir / project_name)
for paths in TEMPLATES_TO_RENDER:
path = join(*paths)
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths))
render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name))
print(f"New Scrapy project '{project_name}', using template directory "
f"'{self.templates_dir}', created in:")
print(f" {abspath(project_dir)}\n")
print(f" {project_dir.resolve()}\n")
print("You can start your first spider with:")
print(f" cd {project_dir}")
print(" scrapy genspider example example.com")
@property
def templates_dir(self):
return join(
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
def templates_dir(self) -> str:
return str(Path(
self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'),
'project'
)
))

5
scrapy/core/downloader/handlers/file.py
View File

@ -1,3 +1,5 @@
from pathlib import Path
from w3lib.url import file_uri_to_path
from scrapy.responsetypes import responsetypes
@ -10,7 +12,6 @@ class FileDownloadHandler:
@defers
def download_request(self, request, spider):
filepath = file_uri_to_path(request.url)
with open(filepath, 'rb') as fo:
body = fo.read()
body = Path(filepath).read_bytes()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)

19
scrapy/core/scheduler.py
View File

@ -1,8 +1,7 @@
import json
import logging
import os
from abc import abstractmethod
from os.path import exists, join
from pathlib import Path
from typing import Optional, Type, TypeVar
from twisted.internet.defer import Deferred
@ -324,19 +323,19 @@ class Scheduler(BaseScheduler):
def _dqdir(self, jobdir: Optional[str]) -> Optional[str]:
""" Return a folder name to keep disk queue state at """
if jobdir is not None:
dqdir = join(jobdir, 'requests.queue')
if not exists(dqdir):
os.makedirs(dqdir)
return dqdir
dqdir = Path(jobdir, 'requests.queue')
if not dqdir.exists():
dqdir.mkdir(parents=True)
return str(dqdir)
return None
def _read_dqs_state(self, dqdir: str) -> list:
path = join(dqdir, 'active.json')
if not exists(path):
path = Path(dqdir, 'active.json')
if not path.exists():
return []
with open(path) as f:
with path.open() as f:
return json.load(f)
def _write_dqs_state(self, dqdir: str, state: list) -> None:
with open(join(dqdir, 'active.json'), 'w') as f:
with Path(dqdir, 'active.json').open('w') as f:
json.dump(state, f)

8
scrapy/crawler.py
View File

@ -1,7 +1,10 @@
from __future__ import annotations
import logging
import pprint
import signal
import warnings
from typing import TYPE_CHECKING
from twisted.internet import defer
from zope.interface.exceptions import DoesNotImplement
@ -33,6 +36,9 @@ from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import install_reactor, verify_installed_reactor
if TYPE_CHECKING:
from scrapy.utils.request import RequestFingerprinter
logger = logging.getLogger(__name__)
@ -72,7 +78,7 @@ class Crawler:
lf_cls = load_object(self.settings['LOG_FORMATTER'])
self.logformatter = lf_cls.from_crawler(self)
self.request_fingerprinter = create_instance(
self.request_fingerprinter: RequestFingerprinter = create_instance(
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
settings=self.settings,
crawler=self,

4
scrapy/dupefilters.py
View File

@ -1,5 +1,5 @@
import logging
import os
from pathlib import Path
from typing import Optional, Set, Type, TypeVar
from warnings import warn
@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug
self.logger = logging.getLogger(__name__)
if path:
self.file = open(os.path.join(path, 'requests.seen'), 'a+')
self.file = Path(path, 'requests.seen').open('a+')
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)

16
scrapy/extensions/feedexport.py
View File

@ -5,13 +5,13 @@ See documentation in docs/topics/feed-exports.rst
"""
import logging
import os
import re
import sys
import warnings
from datetime import datetime
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any, Callable, Optional, Tuple, Union
from typing import IO, Any, Callable, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from twisted.internet import defer, threads
@ -101,7 +101,7 @@ class BlockingFeedStorage:
def open(self, spider):
path = spider.crawler.settings['FEED_TEMPDIR']
if path and not os.path.isdir(path):
if path and not Path(path).is_dir():
raise OSError('Not a Directory: ' + str(path))
return NamedTemporaryFile(prefix='feed-', dir=path)
@ -141,11 +141,11 @@ class FileFeedStorage:
feed_options = feed_options or {}
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
def open(self, spider):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
return open(self.path, self.write_mode)
def open(self, spider) -> IO[Any]:
dirname = Path(self.path).parent
if dirname and not dirname.exists():
dirname.mkdir(parents=True)
return Path(self.path).open(self.write_mode)
def store(self, file):
file.close()

48
scrapy/extensions/httpcache.py
View File

@ -1,16 +1,18 @@
import gzip
import logging
import os
import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
from time import time
from weakref import WeakKeyDictionary
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response
from scrapy.http.request import Request
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.project import data_path
from scrapy.utils.python import to_bytes, to_unicode
@ -221,9 +223,9 @@ class DbmCacheStorage:
self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
self.db = None
def open_spider(self, spider):
dbpath = os.path.join(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(dbpath, 'c')
def open_spider(self, spider: Spider):
dbpath = Path(self.cachedir, f'{spider.name}.db')
self.db = self.dbmodule.open(str(dbpath), 'c')
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
@ -277,7 +279,7 @@ class FilesystemCacheStorage:
self.use_gzip = settings.getbool('HTTPCACHE_GZIP')
self._open = gzip.open if self.use_gzip else open
def open_spider(self, spider):
def open_spider(self, spider: Spider):
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
extra={'spider': spider})
@ -286,15 +288,15 @@ class FilesystemCacheStorage:
def close_spider(self, spider):
pass
def retrieve_response(self, spider, request):
def retrieve_response(self, spider: Spider, request: Request):
"""Return response if present in cache, or None otherwise."""
metadata = self._read_meta(spider, request)
if metadata is None:
return # not cached
rpath = self._get_request_path(spider, request)
with self._open(os.path.join(rpath, 'response_body'), 'rb') as f:
with self._open(rpath / 'response_body', 'rb') as f:
body = f.read()
with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f:
with self._open(rpath / 'response_headers', 'rb') as f:
rawheaders = f.read()
url = metadata.get('response_url')
status = metadata['status']
@ -303,11 +305,11 @@ class FilesystemCacheStorage:
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def store_response(self, spider, request, response):
def store_response(self, spider: Spider, request: Request, response):
"""Store the given response in the cache."""
rpath = self._get_request_path(spider, request)
if not os.path.exists(rpath):
os.makedirs(rpath)
if not rpath.exists():
rpath.mkdir(parents=True)
metadata = {
'url': request.url,
'method': request.method,
@ -315,29 +317,29 @@ class FilesystemCacheStorage:
'response_url': response.url,
'timestamp': time(),
}
with self._open(os.path.join(rpath, 'meta'), 'wb') as f:
with self._open(rpath / 'meta', 'wb') as f:
f.write(to_bytes(repr(metadata)))
with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
with self._open(rpath / 'pickled_meta', 'wb') as f:
pickle.dump(metadata, f, protocol=4)
with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f:
with self._open(rpath / 'response_headers', 'wb') as f:
f.write(headers_dict_to_raw(response.headers))
with self._open(os.path.join(rpath, 'response_body'), 'wb') as f:
with self._open(rpath / 'response_body', 'wb') as f:
f.write(response.body)
with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f:
with self._open(rpath / 'request_headers', 'wb') as f:
f.write(headers_dict_to_raw(request.headers))
with self._open(os.path.join(rpath, 'request_body'), 'wb') as f:
with self._open(rpath / 'request_body', 'wb') as f:
f.write(request.body)
def _get_request_path(self, spider, request):
def _get_request_path(self, spider: Spider, request: Request) -> Path:
key = self._fingerprinter.fingerprint(request).hex()
return os.path.join(self.cachedir, spider.name, key[0:2], key)
return Path(self.cachedir, spider.name, key[0:2], key)
def _read_meta(self, spider, request):
def _read_meta(self, spider: Spider, request: Request):
rpath = self._get_request_path(spider, request)
metapath = os.path.join(rpath, 'pickled_meta')
if not os.path.exists(metapath):
metapath = rpath / 'pickled_meta'
if not metapath.exists():
return # not found
mtime = os.stat(metapath).st_mtime
mtime = metapath.stat().st_mtime
if 0 < self.expiration_secs < time() - mtime:
return # expired
with self._open(metapath, 'rb') as f:

12
scrapy/extensions/spiderstate.py
View File

@ -1,5 +1,5 @@
import os
import pickle
from pathlib import Path
from scrapy import signals
from scrapy.exceptions import NotConfigured
@ -25,16 +25,16 @@ class SpiderState:
def spider_closed(self, spider):
if self.jobdir:
with open(self.statefn, 'wb') as f:
with Path(self.statefn).open('wb') as f:
pickle.dump(spider.state, f, protocol=4)
def spider_opened(self, spider):
if self.jobdir and os.path.exists(self.statefn):
with open(self.statefn, 'rb') as f:
if self.jobdir and Path(self.statefn).exists():
with Path(self.statefn).open('rb') as f:
spider.state = pickle.load(f)
else:
spider.state = {}
@property
def statefn(self):
return os.path.join(self.jobdir, 'spider.state')
def statefn(self) -> str:
return str(Path(self.jobdir, 'spider.state'))

4
scrapy/http/response/__init__.py
View File

@ -32,7 +32,7 @@ class Response(object_ref):
def __init__(
self,
url,
url: str,
status=200,
headers=None,
body=b"",
@ -75,7 +75,7 @@ class Response(object_ref):
def _get_url(self):
return self._url
def _set_url(self, url):
def _set_url(self, url: str):
if isinstance(url, str):
self._url = url
else:

41
scrapy/pipelines/files.py
View File

@ -13,6 +13,8 @@ from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
from io import BytesIO
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse
from itemadapter import ItemAdapter
@ -39,41 +41,40 @@ class FileException(Exception):
class FSFilesStore:
def __init__(self, basedir):
def __init__(self, basedir: str):
if '://' in basedir:
basedir = basedir.split('://', 1)[1]
self.basedir = basedir
self._mkdir(self.basedir)
self.created_directories = defaultdict(set)
self._mkdir(Path(self.basedir))
self.created_directories: defaultdict[str, set[str]] = defaultdict(set)
def persist_file(self, path, buf, info, meta=None, headers=None):
def persist_file(self, path: str, buf, info, meta=None, headers=None):
absolute_path = self._get_filesystem_path(path)
self._mkdir(os.path.dirname(absolute_path), info)
with open(absolute_path, 'wb') as f:
f.write(buf.getvalue())
self._mkdir(absolute_path.parent, info)
absolute_path.write_bytes(buf.getvalue())
def stat_file(self, path, info):
def stat_file(self, path: str, info):
absolute_path = self._get_filesystem_path(path)
try:
last_modified = os.path.getmtime(absolute_path)
last_modified = absolute_path.stat().st_mtime
except os.error:
return {}
with open(absolute_path, 'rb') as f:
with absolute_path.open('rb') as f:
checksum = md5sum(f)
return {'last_modified': last_modified, 'checksum': checksum}
def _get_filesystem_path(self, path):
def _get_filesystem_path(self, path: str) -> Path:
path_comps = path.split('/')
return os.path.join(self.basedir, *path_comps)
return Path(self.basedir, *path_comps)
def _mkdir(self, dirname, domain=None):
def _mkdir(self, dirname: Path, domain: Optional[str] = None):
seen = self.created_directories[domain] if domain else set()
if dirname not in seen:
if not os.path.exists(dirname):
os.makedirs(dirname)
seen.add(dirname)
if str(dirname) not in seen:
if not dirname.exists():
dirname.mkdir(parents=True)
seen.add(str(dirname))
class S3FilesStore:
@ -374,8 +375,8 @@ class FilesPipeline(MediaPipeline):
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)
def _get_store(self, uri):
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir
def _get_store(self, uri: str):
if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir
scheme = 'file'
else:
scheme = urlparse(uri).scheme
@ -510,7 +511,7 @@ class FilesPipeline(MediaPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
media_ext = os.path.splitext(request.url)[1]
media_ext = Path(request.url).suffix
# Handles empty and wild extensions by trying to guess the
# mime type then extension or default to empty string otherwise
if media_ext not in mimetypes.types_map:

4
scrapy/settings/default_settings.py
View File

@ -15,7 +15,7 @@ Scrapy developers, if you add a setting here remember to:
import sys
from importlib import import_module
from os.path import join, abspath, dirname
from pathlib import Path
AJAXCRAWL_ENABLED = False
@ -288,7 +288,7 @@ STATS_DUMP = True
STATSMAILER_RCPTS = []
TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve())
URLLENGTH_LIMIT = 2083

9
scrapy/spiders/__init__.py
View File

@ -3,14 +3,19 @@ Base class for Scrapy spiders
See documentation in docs/topics/spiders.rst
"""
from __future__ import annotations
import logging
from typing import Optional
from typing import TYPE_CHECKING, Optional
from scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class Spider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this
@ -49,7 +54,7 @@ class Spider(object_ref):
spider._set_crawler(crawler)
return spider
def _set_crawler(self, crawler):
def _set_crawler(self, crawler: Crawler):
self.crawler = crawler
self.settings = crawler.settings
crawler.signals.connect(self.close, signals.spider_closed)

11
scrapy/squeues.py
View File

@ -3,8 +3,9 @@ Scheduler queues
"""
import marshal
import os
import pickle
from os import PathLike
from pathlib import Path
from queuelib import queue
@ -16,10 +17,10 @@ def _with_mkdir(queue_class):
class DirectoriesCreated(queue_class):
def __init__(self, path, *args, **kwargs):
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, exist_ok=True)
def __init__(self, path: str | PathLike[str], *args, **kwargs):
dirname = Path(path).parent
if not dirname.exists():
dirname.mkdir(parents=True, exist_ok=True)
super().__init__(path, *args, **kwargs)
return DirectoriesCreated

32
scrapy/utils/conf.py
View File

@ -4,6 +4,8 @@ import sys
import warnings
from configparser import ConfigParser
from operator import itemgetter
from pathlib import Path
from typing import Optional
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
@ -65,17 +67,17 @@ def arglist_to_dict(arglist):
return dict(x.split('=', 1) for x in arglist)
def closest_scrapy_cfg(path='.', prevpath=None):
def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str:
"""Return the path to the closest scrapy.cfg file by traversing the current
directory and its parents
"""
if path == prevpath:
if prevpath is not None and str(path) == str(prevpath):
return ''
path = os.path.abspath(path)
cfgfile = os.path.join(path, 'scrapy.cfg')
if os.path.exists(cfgfile):
return cfgfile
return closest_scrapy_cfg(os.path.dirname(path), path)
path = Path(path).resolve()
cfgfile = path / 'scrapy.cfg'
if cfgfile.exists():
return str(cfgfile)
return closest_scrapy_cfg(path.parent, path)
def init_env(project='default', set_syspath=True):
@ -88,7 +90,7 @@ def init_env(project='default', set_syspath=True):
os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)
closest = closest_scrapy_cfg()
if closest:
projdir = os.path.dirname(closest)
projdir = str(Path(closest).parent)
if set_syspath and projdir not in sys.path:
sys.path.append(projdir)
@ -101,13 +103,13 @@ def get_config(use_closest=True):
return cfg
def get_sources(use_closest=True):
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
def get_sources(use_closest=True) -> list[str]:
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser()
sources = [
'/etc/scrapy.cfg',
r'c:\scrapy\scrapy.cfg',
xdg_config_home + '/scrapy.cfg',
os.path.expanduser('~/.scrapy.cfg'),
str(Path(xdg_config_home) / 'scrapy.cfg'),
str(Path('~/.scrapy.cfg').expanduser()),
]
if use_closest:
sources.append(closest_scrapy_cfg())
@ -129,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings):
return out
def feed_process_params_from_cli(settings, output, output_format=None,
overwrite_output=None):
def feed_process_params_from_cli(settings, output: list[str], output_format=None,
overwrite_output: Optional[list[str]] = None):
"""
Receives feed export params (from the 'crawl' or 'runspider' commands),
checks for inconsistencies in their quantities and returns a dictionary
@ -180,7 +182,7 @@ def feed_process_params_from_cli(settings, output, output_format=None,
feed_uri, feed_format = element.rsplit(':', 1)
except ValueError:
feed_uri = element
feed_format = os.path.splitext(element)[1].replace('.', '')
feed_format = Path(element).suffix.replace('.', '')
else:
if feed_uri == '-':
feed_uri = 'stdout:'

6
scrapy/utils/job.py
View File

@ -1,4 +1,4 @@
import os
from pathlib import Path
from typing import Optional
from scrapy.settings import BaseSettings
@ -6,6 +6,6 @@ from scrapy.settings import BaseSettings
def job_dir(settings: BaseSettings) -> Optional[str]:
path = settings['JOBDIR']
if path and not os.path.exists(path):
os.makedirs(path)
if path and not Path(path).exists():
Path(path).mkdir(parents=True)
return path

29
scrapy/utils/project.py
View File

@ -2,7 +2,7 @@ import os
import warnings
from importlib import import_module
from os.path import join, dirname, abspath, isabs, exists
from pathlib import Path
from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
from scrapy.settings import Settings
@ -25,36 +25,37 @@ def inside_project():
return bool(closest_scrapy_cfg())
def project_data_dir(project='default'):
def project_data_dir(project='default') -> str:
"""Return the current project data dir, creating it if it doesn't exist"""
if not inside_project():
raise NotConfigured("Not inside a project")
cfg = get_config()
if cfg.has_option(DATADIR_CFG_SECTION, project):
d = cfg.get(DATADIR_CFG_SECTION, project)
d = Path(cfg.get(DATADIR_CFG_SECTION, project))
else:
scrapy_cfg = closest_scrapy_cfg()
if not scrapy_cfg:
raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
d = abspath(join(dirname(scrapy_cfg), '.scrapy'))
if not exists(d):
os.makedirs(d)
return d
d = (Path(scrapy_cfg).parent / '.scrapy').resolve()
if not d.exists():
d.mkdir(parents=True)
return str(d)
def data_path(path, createdir=False):
def data_path(path: str, createdir=False) -> str:
"""
Return the given path joined with the .scrapy data directory.
If given an absolute path, return it unmodified.
"""
if not isabs(path):
path_obj = Path(path)
if not path_obj.is_absolute():
if inside_project():
path = join(project_data_dir(), path)
path_obj = Path(project_data_dir(), path)
else:
path = join('.scrapy', path)
if createdir and not exists(path):
os.makedirs(path)
return path
path_obj = Path('.scrapy', path)
if createdir and not path_obj.exists():
path_obj.mkdir(parents=True)
return str(path_obj)
def get_project_settings():

2
scrapy/utils/request.py
View File

@ -264,7 +264,7 @@ class RequestFingerprinter:
f'and \'VERSION\'.'
)
def fingerprint(self, request):
def fingerprint(self, request: Request):
return self._fingerprint(request)

12
scrapy/utils/template.py
View File

@ -1,23 +1,21 @@
"""Helper functions for working with templates"""
import os
import re
import string
from pathlib import Path
def render_templatefile(path, **kwargs):
with open(path, 'rb') as fp:
raw = fp.read().decode('utf8')
def render_templatefile(path: str, **kwargs):
raw = Path(path).read_text('utf8')
content = string.Template(raw).substitute(**kwargs)
render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path
if path.endswith('.tmpl'):
os.rename(path, render_path)
Path(path).rename(render_path)
with open(render_path, 'wb') as fp:
fp.write(content.encode('utf8'))
Path(render_path).write_text(content, 'utf8')
CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')

5
scrapy/utils/test.py
View File

@ -4,6 +4,7 @@ This module contains some assorted functions used in tests
import asyncio
import os
from pathlib import Path
from posixpath import split
from unittest import mock
@ -71,11 +72,11 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
return runner.create_crawler(spidercls or Spider)
def get_pythonpath():
def get_pythonpath() -> str:
"""Return a PYTHONPATH suitable to use in processes so that they find this
installation of Scrapy"""
scrapy_path = import_module('scrapy').__path__[0]
return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '')
def get_testenv():

5
setup.py
View File

@ -1,10 +1,9 @@
from os.path import dirname, join
from pathlib import Path
from pkg_resources import parse_version
from setuptools import setup, find_packages, __version__ as setuptools_version
with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f:
version = f.read().decode('ascii').strip()
version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip()
def has_environment_marker_platform_impl_support():

17
tests/__init__.py
View File

@ -6,6 +6,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests
import os
import socket
from pathlib import Path
# ignore system-wide proxies for tests
# which would send requests to a totally unsuspecting server
@ -16,14 +17,12 @@ os.environ['ftp_proxy'] = ''
# Absolutize paths to coverage config and output file because tests that
# spawn subprocesses also changes current working directory.
_sourceroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_sourceroot = Path(__file__).resolve().parent.parent
if 'COV_CORE_CONFIG' in os.environ:
os.environ['COVERAGE_FILE'] = os.path.join(_sourceroot, '.coverage')
os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
os.environ['COV_CORE_CONFIG'])
os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage')
os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG'])
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'sample_data')
tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data')
# In some environments accessing a non-existing host doesn't raise an
@ -35,8 +34,6 @@ except socket.gaierror:
NON_EXISTING_RESOLVABLE = False
def get_testdata(*paths):
def get_testdata(*paths: str) -> bytes:
"""Return test data"""
path = os.path.join(tests_datadir, *paths)
with open(path, 'rb') as f:
return f.read()
return Path(tests_datadir, *paths).read_bytes()

22
tests/keys/__init__.py
View File

@ -1,5 +1,5 @@
import os
from datetime import datetime, timedelta
from pathlib import Path
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
@ -22,21 +22,20 @@ from cryptography.x509.oid import NameOID
# https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate
def generate_keys():
folder = os.path.dirname(__file__)
folder = Path(__file__).parent
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend(),
)
with open(os.path.join(folder, 'localhost.key'), "wb") as f:
f.write(
key.private_bytes(
encoding=Encoding.PEM,
format=PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=NoEncryption(),
)
)
(folder / 'localhost.key').write_bytes(
key.private_bytes(
encoding=Encoding.PEM,
format=PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=NoEncryption(),
),
)
subject = issuer = Name(
[
@ -59,5 +58,4 @@ def generate_keys():
)
.sign(key, SHA256(), default_backend())
)
with open(os.path.join(folder, 'localhost.crt'), "wb") as f:
f.write(cert.public_bytes(Encoding.PEM))
(folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM))

7
tests/mockserver.py
View File

@ -1,6 +1,5 @@
import argparse
import json
import os
import random
import sys
from pathlib import Path
@ -253,7 +252,7 @@ class Root(resource.Resource):
self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
try:
from tests import tests_datadir
self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
self.putChild(b"files", File(str(Path(tests_datadir, 'test_site/files/'))))
except Exception:
pass
self.putChild(b"redirect-to", RedirectTo())
@ -346,8 +345,8 @@ class MockFTPServer:
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
factory = ssl.DefaultOpenSSLContextFactory(
os.path.join(os.path.dirname(__file__), keyfile),
os.path.join(os.path.dirname(__file__), certfile),
str(Path(__file__).parent / keyfile),
str(Path(__file__).parent / certfile),
)
if cipher_string:
ctx = factory.getContext()

14
tests/test_cmdline/__init__.py
View File

@ -1,11 +1,11 @@
import json
import os
import pstats
import shutil
import sys
import tempfile
import unittest
from io import StringIO
from pathlib import Path
from subprocess import Popen, PIPE
from scrapy.utils.test import get_testenv
@ -36,17 +36,17 @@ class CmdlineTest(unittest.TestCase):
self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
def test_profiling(self):
path = tempfile.mkdtemp()
filename = os.path.join(path, 'res.prof')
path = Path(tempfile.mkdtemp())
filename = path / 'res.prof'
try:
self._execute('version', '--profile', filename)
self.assertTrue(os.path.exists(filename))
self._execute('version', '--profile', str(filename))
self.assertTrue(filename.exists())
out = StringIO()
stats = pstats.Stats(filename, stream=out)
stats = pstats.Stats(str(filename), stream=out)
stats.print_stats()
out.seek(0)
stats = out.read()
self.assertIn(os.path.join('scrapy', 'commands', 'version.py'),
self.assertIn(str(Path('scrapy', 'commands', 'version.py')),
stats)
self.assertIn('tottime', stats)
finally:

4
tests/test_cmdline_crawl_with_pipeline/__init__.py
View File

@ -1,6 +1,6 @@
import os
import sys
import unittest
from pathlib import Path
from subprocess import Popen, PIPE
@ -8,7 +8,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase):
def _execute(self, spname):
args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname)
cwd = os.path.dirname(os.path.abspath(__file__))
cwd = Path(__file__).resolve().parent
proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
proc.communicate()
return proc.returncode

9
tests/test_command_check.py
View File

@ -1,5 +1,3 @@
from os.path import join, abspath
from tests.test_commands import CommandTest
@ -10,11 +8,10 @@ class CheckCommandTest(CommandTest):
def setUp(self):
super(CheckCommandTest, self).setUp()
self.spider_name = 'check_spider'
self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))
self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve()
def _write_contract(self, contracts, parse_def):
with open(self.spider, 'w') as file:
file.write(f"""
self.spider.write_text(f"""
import scrapy
class CheckSpider(scrapy.Spider):
@ -27,7 +24,7 @@ class CheckSpider(scrapy.Spider):
{contracts}
\"\"\"
{parse_def}
""")
""")
def _test_contract(self, contracts='', parse_def='pass'):
self._write_contract(contracts, parse_def)

22
tests/test_command_parse.py
View File

@ -1,6 +1,6 @@
import os
import argparse
from os.path import join, abspath, isfile, exists
from pathlib import Path
from twisted.internet import defer
from scrapy.commands import parse
@ -23,9 +23,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
def setUp(self):
super().setUp()
self.spider_name = 'parse_spider'
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
with open(fname, 'w') as f:
f.write(f"""
(self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f"""
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
@ -88,9 +86,7 @@ class MyBadCrawlSpider(CrawlSpider):
return [scrapy.Item(), dict(foo='bar')]
""")
fname = abspath(join(self.proj_mod_path, 'pipelines.py'))
with open(fname, 'w') as f:
f.write("""
(self.proj_mod_path / 'pipelines.py').write_text("""
import logging
class MyPipeline:
@ -101,8 +97,7 @@ class MyPipeline:
return item
""")
fname = abspath(join(self.proj_mod_path, 'settings.py'))
with open(fname, 'a') as f:
with (self.proj_mod_path / 'settings.py').open("a") as f:
f.write(f"""
ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
""")
@ -234,7 +229,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
correct format containing correct data in it.
"""
file_name = 'data.json'
file_path = join(self.proj_path, file_name)
file_path = Path(self.proj_path, file_name)
yield self.execute([
'--spider', self.spider_name,
'-c', 'parse',
@ -242,12 +237,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.url('/html')
])
self.assertTrue(exists(file_path))
self.assertTrue(isfile(file_path))
self.assertTrue(file_path.exists())
self.assertTrue(file_path.is_file())
content = '[\n{},\n{"foo": "bar"}\n]'
with open(file_path, 'r') as f:
self.assertEqual(f.read(), content)
self.assertEqual(file_path.read_text(), content)
def test_parse_add_options(self):
command = parse.Command()

6
tests/test_command_shell.py
View File

@ -1,4 +1,4 @@
from os.path import join
from pathlib import Path
from twisted.trial import unittest
from twisted.internet import defer
@ -96,8 +96,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_local_file(self):
filepath = join(tests_datadir, 'test_site', 'index.html')
_, out, _ = yield self.execute([filepath, '-c', 'item'])
filepath = Path(tests_datadir, 'test_site', 'index.html')
_, out, _ = yield self.execute([str(filepath), '-c', 'item'])
assert b'{}' in out
@defer.inlineCallbacks

199
tests/test_commands.py
View File

@ -9,12 +9,12 @@ import sys
import tempfile
from contextlib import contextmanager
from itertools import chain
from os.path import exists, join, abspath, getmtime
from pathlib import Path
from shutil import rmtree, copytree
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
from tempfile import mkdtemp
from threading import Timer
from typing import Generator, Optional
from unittest import skipIf
from pytest import mark
@ -66,8 +66,8 @@ class ProjectTest(unittest.TestCase):
def setUp(self):
self.temp_path = mkdtemp()
self.cwd = self.temp_path
self.proj_path = join(self.temp_path, self.project_name)
self.proj_mod_path = join(self.proj_path, self.project_name)
self.proj_path = Path(self.temp_path, self.project_name)
self.proj_mod_path = self.proj_path / self.project_name
self.env = get_testenv()
def tearDown(self):
@ -104,10 +104,10 @@ class ProjectTest(unittest.TestCase):
return p, to_unicode(stdout), to_unicode(stderr)
def find_in_file(self, filename, regex):
def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
with open(filename, "r") as f:
with Path(filename).open("r") as f:
for line in f:
match = pattern.search(line)
if match is not None:
@ -122,13 +122,13 @@ class StartprojectTest(ProjectTest):
print(err, file=sys.stderr)
self.assertEqual(p.returncode, 0)
assert exists(join(self.proj_path, 'scrapy.cfg'))
assert exists(join(self.proj_path, 'testproject'))
assert exists(join(self.proj_mod_path, '__init__.py'))
assert exists(join(self.proj_mod_path, 'items.py'))
assert exists(join(self.proj_mod_path, 'pipelines.py'))
assert exists(join(self.proj_mod_path, 'settings.py'))
assert exists(join(self.proj_mod_path, 'spiders', '__init__.py'))
assert Path(self.proj_path, 'scrapy.cfg').exists()
assert Path(self.proj_path, 'testproject').exists()
assert Path(self.proj_mod_path, '__init__.py').exists()
assert Path(self.proj_mod_path, 'items.py').exists()
assert Path(self.proj_mod_path, 'pipelines.py').exists()
assert Path(self.proj_mod_path, 'settings.py').exists()
assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists()
self.assertEqual(1, self.call('startproject', self.project_name))
self.assertEqual(1, self.call('startproject', 'wrong---project---name'))
@ -138,13 +138,13 @@ class StartprojectTest(ProjectTest):
project_dir = mkdtemp()
self.assertEqual(0, self.call('startproject', self.project_name, project_dir))
assert exists(join(abspath(project_dir), 'scrapy.cfg'))
assert exists(join(abspath(project_dir), 'testproject'))
assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py'))
assert exists(join(join(abspath(project_dir), self.project_name), 'items.py'))
assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py'))
assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py'))
assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py'))
assert Path(project_dir, 'scrapy.cfg').exists()
assert Path(project_dir, 'testproject').exists()
assert Path(project_dir, self.project_name, '__init__.py').exists()
assert Path(project_dir, self.project_name, 'items.py').exists()
assert Path(project_dir, self.project_name, 'pipelines.py').exists()
assert Path(project_dir, self.project_name, 'settings.py').exists()
assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists()
self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2'))
@ -158,40 +158,42 @@ class StartprojectTest(ProjectTest):
def test_existing_project_dir(self):
project_dir = mkdtemp()
project_name = self.project_name + '_existing'
project_path = os.path.join(project_dir, project_name)
os.mkdir(project_path)
project_path = Path(project_dir, project_name)
project_path.mkdir()
p, out, err = self.proc('startproject', project_name, cwd=project_dir)
print(out)
print(err, file=sys.stderr)
self.assertEqual(p.returncode, 0)
assert exists(join(abspath(project_path), 'scrapy.cfg'))
assert exists(join(abspath(project_path), project_name))
assert exists(join(join(abspath(project_path), project_name), '__init__.py'))
assert exists(join(join(abspath(project_path), project_name), 'items.py'))
assert exists(join(join(abspath(project_path), project_name), 'pipelines.py'))
assert exists(join(join(abspath(project_path), project_name), 'settings.py'))
assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py'))
assert Path(project_path, 'scrapy.cfg').exists()
assert Path(project_path, project_name).exists()
assert Path(project_path, project_name, '__init__.py').exists()
assert Path(project_path, project_name, 'items.py').exists()
assert Path(project_path, project_name, 'pipelines.py').exists()
assert Path(project_path, project_name, 'settings.py').exists()
assert Path(project_path, project_name, 'spiders', '__init__.py').exists()
def get_permissions_dict(path, renamings=None, ignore=None):
def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]:
def get_permissions(path):
return oct(os.stat(path).st_mode)
def get_permissions(path: Path) -> str:
return oct(path.stat().st_mode)
path_obj = Path(path)
renamings = renamings or tuple()
permissions_dict = {
'.': get_permissions(path),
'.': get_permissions(path_obj),
}
for root, dirs, files in os.walk(path):
for root, dirs, files in os.walk(path_obj):
nodes = list(chain(dirs, files))
if ignore:
ignored_names = ignore(root, nodes)
nodes = [node for node in nodes if node not in ignored_names]
for node in nodes:
absolute_path = os.path.join(root, node)
relative_path = os.path.relpath(absolute_path, path)
absolute_path = Path(root, node)
relative_path = str(absolute_path.relative_to(path))
for search_string, replacement in renamings:
relative_path = relative_path.replace(
search_string,
@ -208,28 +210,27 @@ class StartprojectTemplatesTest(ProjectTest):
def setUp(self):
super().setUp()
self.tmpl = join(self.temp_path, 'templates')
self.tmpl_proj = join(self.tmpl, 'project')
self.tmpl = str(Path(self.temp_path, 'templates'))
self.tmpl_proj = str(Path(self.tmpl, 'project'))
def test_startproject_template_override(self):
copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)
with open(join(self.tmpl_proj, 'root_template'), 'w'):
pass
assert exists(join(self.tmpl_proj, 'root_template'))
copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl)
Path(self.tmpl_proj, 'root_template').write_bytes(b"")
assert Path(self.tmpl_proj, 'root_template').exists()
args = ['--set', f'TEMPLATES_DIR={self.tmpl}']
p, out, err = self.proc('startproject', self.project_name, *args)
self.assertIn(f"New Scrapy project '{self.project_name}', "
"using template directory", out)
self.assertIn(self.tmpl_proj, out)
assert exists(join(self.proj_path, 'root_template'))
assert Path(self.proj_path, 'root_template').exists()
def test_startproject_permissions_from_writable(self):
"""Check that generated files have the right permissions when the
template folder has the same permissions as in the project, i.e.
everything is writable."""
scrapy_path = scrapy.__path__[0]
project_template = os.path.join(scrapy_path, 'templates', 'project')
project_template = Path(scrapy_path, 'templates', 'project')
project_name = 'startproject1'
renamings = (
('module', project_name),
@ -255,7 +256,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
process.wait()
project_dir = os.path.join(destination, project_name)
project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions)
@ -268,8 +269,8 @@ class StartprojectTemplatesTest(ProjectTest):
See https://github.com/scrapy/scrapy/pull/4604
"""
scrapy_path = scrapy.__path__[0]
templates_dir = os.path.join(scrapy_path, 'templates')
project_template = os.path.join(templates_dir, 'project')
templates_dir = Path(scrapy_path, 'templates')
project_template = Path(templates_dir, 'project')
project_name = 'startproject2'
renamings = (
('module', project_name),
@ -281,16 +282,16 @@ class StartprojectTemplatesTest(ProjectTest):
IGNORE,
)
def _make_read_only(path):
current_permissions = os.stat(path).st_mode
os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)
def _make_read_only(path: Path):
current_permissions = path.stat().st_mode
path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION)
read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
copytree(templates_dir, read_only_templates_dir)
for root, dirs, files in os.walk(read_only_templates_dir):
for node in chain(dirs, files):
_make_read_only(os.path.join(root, node))
_make_read_only(Path(root, node))
destination = mkdtemp()
process = subprocess.Popen(
@ -308,7 +309,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
process.wait()
project_dir = os.path.join(destination, project_name)
project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions)
@ -317,7 +318,7 @@ class StartprojectTemplatesTest(ProjectTest):
"""Check that pre-existing folders and files in the destination folder
do not see their permissions modified."""
scrapy_path = scrapy.__path__[0]
project_template = os.path.join(scrapy_path, 'templates', 'project')
project_template = Path(scrapy_path, 'templates', 'project')
project_name = 'startproject3'
renamings = (
('module', project_name),
@ -330,7 +331,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
destination = mkdtemp()
project_dir = os.path.join(destination, project_name)
project_dir = Path(destination, project_name)
existing_nodes = {
oct(permissions)[2:] + extension: permissions
@ -339,10 +340,9 @@ class StartprojectTemplatesTest(ProjectTest):
0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
)
}
os.mkdir(project_dir)
project_dir_path = Path(project_dir)
project_dir.mkdir()
for node, permissions in existing_nodes.items():
path = project_dir_path / node
path = project_dir / node
if node.endswith('.d'):
path.mkdir(mode=permissions)
else:
@ -378,7 +378,7 @@ class StartprojectTemplatesTest(ProjectTest):
os.umask(cur_mask)
scrapy_path = scrapy.__path__[0]
project_template = os.path.join(
project_template = Path(
scrapy_path,
'templates',
'project'
@ -409,7 +409,7 @@ class StartprojectTemplatesTest(ProjectTest):
)
process.wait()
project_dir = os.path.join(destination, project_name)
project_dir = Path(destination, project_name)
actual_permissions = get_permissions_dict(project_dir)
self.assertEqual(actual_permissions, expected_permissions)
@ -420,7 +420,7 @@ class CommandTest(ProjectTest):
def setUp(self):
super().setUp()
self.call('startproject', self.project_name)
self.cwd = join(self.temp_path, self.project_name)
self.cwd = Path(self.temp_path, self.project_name)
self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings'
@ -429,10 +429,10 @@ class GenspiderCommandTest(CommandTest):
def test_arguments(self):
# only pass one argument. spider script shouldn't be created
self.assertEqual(2, self.call('genspider', 'test_name'))
assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
# pass two arguments <name> <domain>. spider script should be created
self.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))
assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists()
def test_template(self, tplname='crawl'):
args = [f'--template={tplname}'] if tplname else []
@ -440,11 +440,11 @@ class GenspiderCommandTest(CommandTest):
spmodule = f"{self.project_name}.spiders.{spname}"
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out)
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists())
modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
p, out, err = self.proc('genspider', spname, 'test.com', *args)
self.assertIn(f"Spider {spname!r} already exists in module", out)
modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
def test_template_basic(self):
@ -465,37 +465,37 @@ class GenspiderCommandTest(CommandTest):
def test_same_name_as_project(self):
self.assertEqual(2, self.call('genspider', self.project_name))
assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py'))
assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists()
def test_same_filename_as_existing_spider(self, force=False):
file_name = 'example'
file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py')
file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py')
self.assertEqual(0, self.call('genspider', file_name, 'example.com'))
assert exists(file_path)
assert file_path.exists()
# change name of spider but not its file name
with open(file_path, 'r+') as spider_file:
with file_path.open('r+') as spider_file:
file_data = spider_file.read()
file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
spider_file.seek(0)
spider_file.write(file_data)
spider_file.truncate()
modify_time_before = getmtime(file_path)
modify_time_before = file_path.stat().st_mtime
file_contents_before = file_data
if force:
p, out, err = self.proc('genspider', '--force', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out)
modify_time_after = getmtime(file_path)
modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read()
file_contents_after = file_path.read_text()
self.assertNotEqual(file_contents_after, file_contents_before)
else:
p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"{file_path} already exists", out)
modify_time_after = getmtime(file_path)
modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read()
file_contents_after = file_path.read_text()
self.assertEqual(file_contents_after, file_contents_before)
def test_same_filename_as_existing_spider_force(self):
@ -504,11 +504,11 @@ class GenspiderCommandTest(CommandTest):
def test_url(self, url='test.com', domain="test.com"):
self.assertEqual(0, self.call('genspider', '--force', 'test_name', url))
self.assertEqual(domain,
self.find_in_file(join(self.proj_mod_path,
self.find_in_file(Path(self.proj_mod_path,
'spiders', 'test_name.py'),
r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1))
self.assertEqual(f'http://{domain}/',
self.find_in_file(join(self.proj_mod_path,
self.find_in_file(Path(self.proj_mod_path,
'spiders', 'test_name.py'),
r'start_urls\s*=\s*\[\'(.+)\'\]').group(1))
@ -526,31 +526,31 @@ class GenspiderStandaloneCommandTest(ProjectTest):
def test_generate_standalone_spider(self):
self.call('genspider', 'example', 'example.com')
assert exists(join(self.temp_path, 'example.py'))
assert Path(self.temp_path, 'example.py').exists()
def test_same_name_as_existing_file(self, force=False):
file_name = 'example'
file_path = join(self.temp_path, file_name + '.py')
file_path = Path(self.temp_path, file_name + '.py')
p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out)
assert exists(file_path)
modify_time_before = getmtime(file_path)
file_contents_before = open(file_path, 'r').read()
assert file_path.exists()
modify_time_before = file_path.stat().st_mtime
file_contents_before = file_path.read_text()
if force:
# use different template to ensure contents were changed
p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')
self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out)
modify_time_after = getmtime(file_path)
modify_time_after = file_path.stat().st_mtime
self.assertNotEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read()
file_contents_after = file_path.read_text()
self.assertNotEqual(file_contents_after, file_contents_before)
else:
p, out, err = self.proc('genspider', file_name, 'example.com')
self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out)
modify_time_after = getmtime(file_path)
self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out)
modify_time_after = file_path.stat().st_mtime
self.assertEqual(modify_time_after, modify_time_before)
file_contents_after = open(file_path, 'r').read()
file_contents_after = file_path.read_text()
self.assertEqual(file_contents_after, file_contents_before)
def test_same_name_as_existing_file_force(self):
@ -588,17 +588,16 @@ class BadSpider(scrapy.Spider):
"""
@contextmanager
def _create_file(self, content, name=None):
tmpdir = self.mktemp()
os.mkdir(tmpdir)
def _create_file(self, content, name=None) -> Generator[str, None, None]:
tmpdir = Path(self.mktemp())
tmpdir.mkdir()
if name:
fname = abspath(join(tmpdir, name))
fname = (tmpdir / name).resolve()
else:
fname = abspath(join(tmpdir, self.spider_filename))
with open(fname, 'w') as f:
f.write(content)
fname = (tmpdir / self.spider_filename).resolve()
fname.write_text(content)
try:
yield fname
yield str(fname)
finally:
rmtree(tmpdir)
@ -747,12 +746,11 @@ class MySpider(scrapy.Spider):
)
return []
"""
with open(os.path.join(self.cwd, "example.json"), "w") as f1:
f1.write("not empty")
Path(self.cwd, "example.json").write_text("not empty")
args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
with open(os.path.join(self.cwd, "example.json")) as f2:
with Path(self.cwd, "example.json").open() as f2:
first_line = f2.readline()
self.assertNotEqual(first_line, "not empty")
@ -854,9 +852,7 @@ class ViewCommandTest(CommandTest):
class CrawlCommandTest(CommandTest):
def crawl(self, code, args=()):
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
with open(fname, 'w') as f:
f.write(code)
Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code)
return self.proc('crawl', 'myspider', *args)
def get_log(self, code, args=()):
@ -908,12 +904,11 @@ class MySpider(scrapy.Spider):
)
return []
"""
with open(os.path.join(self.cwd, "example.json"), "w") as f1:
f1.write("not empty")
Path(self.cwd, "example.json").write_text("not empty")
args = ['-O', 'example.json']
log = self.get_log(spider_code, args=args)
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
with open(os.path.join(self.cwd, "example.json")) as f2:
with Path(self.cwd, "example.json").open() as f2:
first_line = f2.readline()
self.assertNotEqual(first_line, "not empty")

32
tests/test_crawler.py
View File

@ -1,9 +1,9 @@
import logging
import os
import platform
import subprocess
import sys
import warnings
from pathlib import Path
from pytest import raises, mark
from twisted import version as twisted_version
@ -100,15 +100,14 @@ class CrawlerLoggingTestCase(unittest.TestCase):
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
log_file = self.mktemp()
with open(log_file, 'wb') as fo:
fo.write('previous message\n'.encode('utf-8'))
log_file = Path(self.mktemp())
log_file.write_text('previous message\n', encoding='utf-8')
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
'LOG_LEVEL': 'INFO',
'LOG_FILE': log_file,
'LOG_FILE': str(log_file),
# settings to avoid extra warnings
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
@ -124,8 +123,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
logging.warning('warning message')
logging.error('error message')
with open(log_file, 'rb') as fo:
logged = fo.read().decode('utf-8')
logged = log_file.read_text(encoding='utf-8')
self.assertIn('previous message', logged)
self.assertNotIn('debug message', logged)
@ -139,14 +137,13 @@ class CrawlerLoggingTestCase(unittest.TestCase):
self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0)
def test_spider_custom_settings_log_append(self):
log_file = self.mktemp()
with open(log_file, 'wb') as fo:
fo.write('previous message\n'.encode('utf-8'))
log_file = Path(self.mktemp())
log_file.write_text('previous message\n', encoding='utf-8')
class MySpider(scrapy.Spider):
name = 'spider'
custom_settings = {
'LOG_FILE': log_file,
'LOG_FILE': str(log_file),
'LOG_FILE_APPEND': False,
# disable telnet if not available to avoid an extra warning
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
@ -156,8 +153,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
get_crawler(MySpider)
logging.debug('debug message')
with open(log_file, 'rb') as fo:
logged = fo.read().decode('utf-8')
logged = log_file.read_text(encoding='utf-8')
self.assertNotIn('previous message', logged)
self.assertIn('debug message', logged)
@ -296,9 +292,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
class ScriptRunnerMixin:
def run_script(self, script_name, *script_args):
script_path = os.path.join(self.script_dir, script_name)
args = [sys.executable, script_path] + list(script_args)
def run_script(self, script_name: str, *script_args):
script_path = self.script_dir / script_name
args = [sys.executable, str(script_path)] + list(script_args)
p = subprocess.Popen(args, env=get_testenv(),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
@ -306,7 +302,7 @@ class ScriptRunnerMixin:
class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess'
def test_simple(self):
log = self.run_script('simple.py')
@ -463,7 +459,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner'
def test_response_ip_address(self):
log = self.run_script("ip_address.py")

7
tests/test_dependencies.py
View File

@ -2,6 +2,7 @@ import os
import re
from configparser import ConfigParser
from importlib import import_module
from pathlib import Path
from twisted import version as twisted_version
from twisted.trial import unittest
@ -29,11 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase):
if not os.environ.get('_SCRAPY_PINNED', None):
self.skipTest('Not in a pinned environment')
tox_config_file_path = os.path.join(
os.path.dirname(__file__),
'..',
'tox.ini',
)
tox_config_file_path = Path(__file__) / '..' / 'tox.ini'
config_parser = ConfigParser()
config_parser.read(tox_config_file_path)
pattern = r'Twisted\[http2\]==([\d.]+)'

69
tests/test_downloader_handlers.py
View File

@ -3,6 +3,7 @@ import os
import shutil
import sys
import tempfile
from pathlib import Path
from typing import Optional, Type
from unittest import mock, SkipTest
@ -10,7 +11,6 @@ from testfixtures import LogCapture
from twisted.cred import checkers, credentials, portal
from twisted.internet import defer, error, reactor
from twisted.protocols.policies import WrappingFactory
from twisted.python.filepath import FilePath
from twisted.trial import unittest
from twisted.web import resource, server, static, util
from twisted.web._newclient import ResponseFailed
@ -108,14 +108,13 @@ class LoadTestCase(unittest.TestCase):
class FileTestCase(unittest.TestCase):
def setUp(self):
self.tmpname = self.mktemp()
with open(self.tmpname + '^', 'w') as f:
f.write('0123456789')
self.tmpname = Path(self.mktemp() + '^')
Path(self.tmpname).write_text('0123456789')
handler = create_instance(FileDownloadHandler, None, get_crawler())
self.download_request = handler.download_request
def tearDown(self):
os.unlink(self.tmpname + '^')
self.tmpname.unlink()
def test_download(self):
def _test(response):
@ -124,7 +123,7 @@ class FileTestCase(unittest.TestCase):
self.assertEqual(response.body, b'0123456789')
self.assertEqual(response.protocol, None)
request = Request(path_to_file_uri(self.tmpname + '^'))
request = Request(path_to_file_uri(str(self.tmpname)))
assert request.url.upper().endswith('%5E')
return self.download_request(request, Spider('foo')).addCallback(_test)
@ -223,10 +222,10 @@ class HttpTestCase(unittest.TestCase):
certfile = 'keys/localhost.crt'
def setUp(self):
self.tmpname = self.mktemp()
os.mkdir(self.tmpname)
FilePath(self.tmpname).child("file").setContent(b"0123456789")
r = static.File(self.tmpname)
self.tmpname = Path(self.mktemp())
self.tmpname.mkdir()
(self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(str(self.tmpname))
r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
@ -626,10 +625,10 @@ class Https11CustomCiphers(unittest.TestCase):
certfile = 'keys/localhost.crt'
def setUp(self):
self.tmpname = self.mktemp()
os.mkdir(self.tmpname)
FilePath(self.tmpname).child("file").setContent(b"0123456789")
r = static.File(self.tmpname)
self.tmpname = Path(self.mktemp())
self.tmpname.mkdir()
(self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(str(self.tmpname))
self.site = server.Site(r, timeout=None)
self.host = 'localhost'
self.port = reactor.listenSSL(
@ -1002,16 +1001,15 @@ class BaseFTPTestCase(unittest.TestCase):
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
# setup dirs and test file
self.directory = self.mktemp()
os.mkdir(self.directory)
userdir = os.path.join(self.directory, self.username)
os.mkdir(userdir)
fp = FilePath(userdir)
self.directory = Path(self.mktemp())
self.directory.mkdir()
userdir = self.directory / self.username
userdir.mkdir()
for filename, content in self.test_files:
fp.child(filename).setContent(content)
(userdir / filename).write_bytes(content)
# setup server
realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory))
p = portal.Portal(realm)
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
users_checker.addUser(self.username, self.password)
@ -1076,28 +1074,28 @@ class BaseFTPTestCase(unittest.TestCase):
def test_ftp_local_filename(self):
f, local_fname = tempfile.mkstemp()
local_fname = to_bytes(local_fname)
fname_bytes = to_bytes(local_fname)
local_fname = Path(local_fname)
os.close(f)
meta = {"ftp_local_filename": local_fname}
meta = {"ftp_local_filename": fname_bytes}
meta.update(self.req_meta)
request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt",
meta=meta)
d = self.download_handler.download_request(request, None)
def _test(r):
self.assertEqual(r.body, local_fname)
self.assertEqual(r.headers, {b'Local Filename': [local_fname],
self.assertEqual(r.body, fname_bytes)
self.assertEqual(r.headers, {b'Local Filename': [fname_bytes],
b'Size': [b'17']})
self.assertTrue(os.path.exists(local_fname))
with open(local_fname, "rb") as f:
self.assertEqual(f.read(), b"I have the power!")
os.remove(local_fname)
self.assertTrue(local_fname.exists())
self.assertEqual(local_fname.read_bytes(), b"I have the power!")
local_fname.unlink()
return self._add_test_callbacks(d, _test)
def _test_response_class(self, filename, response_class):
f, local_fname = tempfile.mkstemp()
local_fname = to_bytes(local_fname)
local_fname = Path(local_fname)
os.close(f)
meta = {}
meta.update(self.req_meta)
@ -1107,7 +1105,7 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r):
self.assertEqual(type(r), response_class)
os.remove(local_fname)
local_fname.unlink()
return self._add_test_callbacks(d, _test)
def test_response_class_from_url(self):
@ -1147,15 +1145,14 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
# setup dir and test file
self.directory = self.mktemp()
os.mkdir(self.directory)
self.directory = Path(self.mktemp())
self.directory.mkdir()
fp = FilePath(self.directory)
for filename, content in self.test_files:
fp.child(filename).setContent(content)
(self.directory / filename).write_bytes(content)
# setup server for anonymous access
realm = FTPRealm(anonymousRoot=self.directory)
realm = FTPRealm(anonymousRoot=str(self.directory))
p = portal.Portal(realm)
p.registerChecker(checkers.AllowAnonymousAccess(),
credentials.IAnonymous)

7
tests/test_downloadermiddleware_httpcompression.py
View File

@ -1,6 +1,6 @@
from gzip import GzipFile
from io import BytesIO
from os.path import join
from pathlib import Path
from unittest import TestCase, SkipTest
from warnings import catch_warnings
@ -15,7 +15,7 @@ from tests import tests_datadir
from w3lib.encoding import resolve_encoding
SAMPLEDIR = join(tests_datadir, 'compressed')
SAMPLEDIR = Path(tests_datadir, 'compressed')
FORMAT = {
'gzip': ('html-gzip.bin', 'gzip'),
@ -46,8 +46,7 @@ class HttpCompressionTest(TestCase):
samplefile, contentencoding = FORMAT[coding]
with open(join(SAMPLEDIR, samplefile), 'rb') as sample:
body = sample.read()
body = (SAMPLEDIR / samplefile).read_bytes()
headers = {
'Server': 'Yaws/1.49 Yet Another Web Server',

4
tests/test_dupefilters.py
View File

@ -2,8 +2,8 @@ import hashlib
import tempfile
import unittest
import shutil
import os
import sys
from pathlib import Path
from testfixtures import LogCapture
from scrapy.dupefilters import RFPDupeFilter
@ -157,7 +157,7 @@ class RFPDupeFilterTest(unittest.TestCase):
df.request_seen(r1)
df.close('finished')
with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file:
with Path(path, 'requests.seen').open('rb') as seen_file:
line = next(seen_file).decode()
assert not line.endswith('\r\r\n')
if sys.platform == 'win32':

6
tests/test_engine.py
View File

@ -10,11 +10,11 @@ module with the ``runserver`` argument::
python test_engine.py runserver
"""
import os
import re
import subprocess
import sys
from collections import defaultdict
from pathlib import Path
from threading import Timer
from urllib.parse import urlparse
from dataclasses import dataclass
@ -127,8 +127,8 @@ class ChangeCloseReasonSpider(TestSpider):
def start_test_site(debug=False):
root_dir = os.path.join(tests_datadir, "test_site")
r = static.File(root_dir)
root_dir = Path(tests_datadir, "test_site")
r = static.File(str(root_dir))
r.putChild(b"redirect", util.Redirect(b"/redirected"))
r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**18)]

155
tests/test_feedexport.py
View File

@ -3,7 +3,6 @@ import csv
import gzip
import json
import lzma
import os
import random
import shutil
import string
@ -15,6 +14,7 @@ from collections import defaultdict
from contextlib import ExitStack
from io import BytesIO
from logging import getLogger
from os import PathLike
from pathlib import Path
from string import ascii_letters, digits
from unittest import mock
@ -63,41 +63,41 @@ def printf_escape(string):
return string.replace('%', '%%')
def build_url(path):
if path[0] != '/':
path = '/' + path
return urljoin('file:', path)
def build_url(path: str | PathLike[str]) -> str:
path_str = str(path)
if path_str[0] != '/':
path_str = '/' + path_str
return urljoin('file:', path_str)
class FileFeedStorageTest(unittest.TestCase):
def test_store_file_uri(self):
path = os.path.abspath(self.mktemp())
uri = path_to_file_uri(path)
path = Path(self.mktemp()).resolve()
uri = path_to_file_uri(str(path))
return self._assert_stores(FileFeedStorage(uri), path)
def test_store_file_uri_makedirs(self):
path = os.path.abspath(self.mktemp())
path = os.path.join(path, 'more', 'paths', 'file.txt')
uri = path_to_file_uri(path)
path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt'
uri = path_to_file_uri(str(path))
return self._assert_stores(FileFeedStorage(uri), path)
def test_store_direct_path(self):
path = os.path.abspath(self.mktemp())
return self._assert_stores(FileFeedStorage(path), path)
path = Path(self.mktemp()).resolve()
return self._assert_stores(FileFeedStorage(str(path)), path)
def test_store_direct_path_relative(self):
path = self.mktemp()
return self._assert_stores(FileFeedStorage(path), path)
path = Path(self.mktemp())
return self._assert_stores(FileFeedStorage(str(path)), path)
def test_interface(self):
path = self.mktemp()
st = FileFeedStorage(path)
verifyObject(IFeedStorage, st)
def _store(self, feed_options=None):
path = os.path.abspath(self.mktemp())
storage = FileFeedStorage(path, feed_options=feed_options)
def _store(self, feed_options=None) -> Path:
path = Path(self.mktemp()).resolve()
storage = FileFeedStorage(str(path), feed_options=feed_options)
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
@ -106,27 +106,26 @@ class FileFeedStorageTest(unittest.TestCase):
def test_append(self):
path = self._store()
return self._assert_stores(FileFeedStorage(path), path, b"contentcontent")
return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent")
def test_overwrite(self):
path = self._store({"overwrite": True})
return self._assert_stores(
FileFeedStorage(path, feed_options={"overwrite": True}),
FileFeedStorage(str(path), feed_options={"overwrite": True}),
path
)
@defer.inlineCallbacks
def _assert_stores(self, storage, path, expected_content=b"content"):
def _assert_stores(self, storage, path: Path, expected_content=b"content"):
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
yield storage.store(file)
self.assertTrue(os.path.exists(path))
self.assertTrue(path.exists())
try:
with open(path, 'rb') as fp:
self.assertEqual(fp.read(), expected_content)
self.assertEqual(path.read_bytes(), expected_content)
finally:
os.unlink(path)
path.unlink()
class FTPFeedStorageTest(unittest.TestCase):
@ -152,13 +151,12 @@ class FTPFeedStorageTest(unittest.TestCase):
file.write(content)
return storage.store(file)
def _assert_stored(self, path, content):
def _assert_stored(self, path: Path, content):
self.assertTrue(path.exists())
try:
with path.open('rb') as fp:
self.assertEqual(fp.read(), content)
self.assertEqual(path.read_bytes(), content)
finally:
os.unlink(str(path))
path.unlink()
@defer.inlineCallbacks
def test_append(self):
@ -221,24 +219,24 @@ class BlockingFeedStorageTest(unittest.TestCase):
b = BlockingFeedStorage()
tmp = b.open(self.get_test_spider())
tmp_path = os.path.dirname(tmp.name)
self.assertEqual(tmp_path, tempfile.gettempdir())
tmp_path = Path(tmp.name).parent
self.assertEqual(str(tmp_path), tempfile.gettempdir())
def test_temp_file(self):
b = BlockingFeedStorage()
tests_path = os.path.dirname(os.path.abspath(__file__))
spider = self.get_test_spider({'FEED_TEMPDIR': tests_path})
tests_path = Path(__file__).resolve().parent
spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)})
tmp = b.open(spider)
tmp_path = os.path.dirname(tmp.name)
tmp_path = Path(tmp.name).parent
self.assertEqual(tmp_path, tests_path)
def test_invalid_folder(self):
b = BlockingFeedStorage()
tests_path = os.path.dirname(os.path.abspath(__file__))
invalid_path = os.path.join(tests_path, 'invalid_path')
spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path})
tests_path = Path(__file__).resolve().parent
invalid_path = tests_path / 'invalid_path'
spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)})
self.assertRaises(OSError, b.open, spider=spider)
@ -564,13 +562,13 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
class DummyBlockingFeedStorage(BlockingFeedStorage):
def __init__(self, uri, *args, feed_options=None):
self.path = file_uri_to_path(uri)
self.path = Path(file_uri_to_path(uri))
def _store_in_thread(self, file):
dirname = os.path.dirname(self.path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
with open(self.path, 'ab') as output_file:
dirname = self.path.parent
if dirname and not dirname.exists():
dirname.mkdir(parents=True)
with self.path.open('ab') as output_file:
output_file.write(file.read())
file.close()
@ -613,10 +611,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):
foo = scrapy.Field()
hello = scrapy.Field()
def _random_temp_filename(self, inter_dir=''):
def _random_temp_filename(self, inter_dir='') -> Path:
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = ''.join(chars)
return os.path.join(self.temp_dir, inter_dir, filename)
return Path(self.temp_dir, inter_dir, filename)
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
@ -702,18 +700,17 @@ class FeedExportTest(FeedExportTestBase):
yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)):
if not Path(file_path).exists():
continue
with open(str(file_path), 'rb') as f:
content[feed_options['format']] = f.read()
content[feed_options['format']] = Path(file_path).read_bytes()
finally:
for file_path in FEEDS.keys():
if not os.path.exists(str(file_path)):
if not Path(file_path).exists():
continue
os.remove(str(file_path))
Path(file_path).unlink()
return content
@ -808,7 +805,7 @@ class FeedExportTest(FeedExportTestBase):
def test_stats_file_success(self):
settings = {
"FEEDS": {
printf_escape(path_to_url(self._random_temp_filename())): {
printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json",
}
},
@ -823,7 +820,7 @@ class FeedExportTest(FeedExportTestBase):
def test_stats_file_failed(self):
settings = {
"FEEDS": {
printf_escape(path_to_url(self._random_temp_filename())): {
printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json",
}
},
@ -846,7 +843,7 @@ class FeedExportTest(FeedExportTestBase):
'AWS_ACCESS_KEY_ID': 'access_key',
'AWS_SECRET_ACCESS_KEY': 'secret_key',
"FEEDS": {
printf_escape(path_to_url(self._random_temp_filename())): {
printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json",
},
"s3://bucket/key/foo.csv": {
@ -1427,12 +1424,11 @@ class FeedExportTest(FeedExportTestBase):
self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler)
@defer.inlineCallbacks
def test_pathlib_uri(self):
feed_path = Path(self._random_temp_filename())
def test_str_uri(self):
settings = {
'FEED_STORE_EMPTY': True,
'FEEDS': {
feed_path: {'format': 'csv'}
str(self._random_temp_filename()): {'format': 'csv'}
},
}
data = yield self.exported_no_data(settings)
@ -1538,8 +1534,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
def close(self):
self.file.close()
def _named_tempfile(self, name):
return os.path.join(self.temp_dir, name)
def _named_tempfile(self, name) -> str:
return str(Path(self.temp_dir, name))
@defer.inlineCallbacks
def run_and_export(self, spider_cls, settings):
@ -1559,18 +1555,17 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)):
if not Path(file_path).exists():
continue
with open(str(file_path), 'rb') as f:
content[str(file_path)] = f.read()
content[str(file_path)] = Path(file_path).read_bytes()
finally:
for file_path in FEEDS.keys():
if not os.path.exists(str(file_path)):
if not Path(file_path).exists():
continue
os.remove(str(file_path))
Path(file_path).unlink()
return content
@ -2031,11 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
yield crawler.crawl()
for path, feed in FEEDS.items():
dir_name = os.path.dirname(path)
for file in sorted(os.listdir(dir_name)):
with open(os.path.join(dir_name, file), 'rb') as f:
data = f.read()
content[feed['format']].append(data)
dir_name = Path(path).parent
for file in sorted(dir_name.iterdir()):
content[feed['format']].append(file.read_bytes())
finally:
self.tearDown()
defer.returnValue(content)
@ -2045,7 +2038,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2061,7 +2054,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2077,7 +2070,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2094,8 +2087,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'},
self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2120,7 +2113,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2137,7 +2130,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = settings or {}
settings.update({
'FEEDS': {
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'},
},
})
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
@ -2184,7 +2177,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
for fmt in ('json', 'jsonlines', 'xml', 'csv'):
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
},
'FEED_EXPORT_BATCH_ITEM_COUNT': 1
}
@ -2204,7 +2197,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
for fmt, expctd in formats:
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},
self._random_temp_filename() / fmt / self._file_mark: {'format': fmt},
},
'FEED_STORE_EMPTY': True,
'FEED_EXPORT_INDENT': None,
@ -2237,19 +2230,19 @@ class BatchDeliveriesTest(FeedExportTestBase):
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
self._random_temp_filename() / 'json' / self._file_mark: {
'format': 'json',
'indent': 0,
'fields': ['bar'],
'encoding': 'utf-8',
},
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {
self._random_temp_filename() / 'xml' / self._file_mark: {
'format': 'xml',
'indent': 2,
'fields': ['foo'],
'encoding': 'latin-1',
},
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
self._random_temp_filename() / 'csv' / self._file_mark: {
'format': 'csv',
'indent': None,
'fields': ['foo', 'bar'],
@ -2272,7 +2265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
}
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
self._random_temp_filename() / 'json' / self._file_mark: {
'format': 'json',
'indent': None,
'encoding': 'utf-8',
@ -2299,7 +2292,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
]
settings = {
'FEEDS': {
os.path.join(self._random_temp_filename(), '%(batch_time)s'): {
self._random_temp_filename() / '%(batch_time)s': {
'format': 'json',
},
},
@ -2312,7 +2305,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
def test_stats_batch_file_success(self):
settings = {
"FEEDS": {
build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): {
build_url(str(self._random_temp_filename() / "json" / self._file_mark)): {
"format": "json",
}
},

15
tests/test_http2_client_protocol.py
View File

@ -1,10 +1,10 @@
import json
import os
import random
import re
import shutil
import string
from ipaddress import IPv4Address
from pathlib import Path
from unittest import mock, skipIf
from urllib.parse import urlencode
@ -163,9 +163,8 @@ class RequestHeaders(LeafResource):
return bytes(json.dumps(headers), 'utf-8')
def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate:
pem = ''.join(key.readlines()) + ''.join(certificate.readlines())
def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate:
pem = key_file.read_text() + certificate_file.read_text()
return PrivateCertificate.loadPEM(pem)
@ -173,12 +172,12 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate:
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2ClientProtocolTestCase(TestCase):
scheme = 'https'
key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key')
certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt')
key_file = Path(__file__).parent / 'keys' / 'localhost.key'
certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt'
def _init_resource(self):
self.temp_directory = self.mktemp()
os.mkdir(self.temp_directory)
Path(self.temp_directory).mkdir()
r = File(self.temp_directory)
r.putChild(b'get-data-html-small', GetDataHtmlSmall())
r.putChild(b'get-data-html-large', GetDataHtmlLarge())
@ -202,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase):
# Start server for testing
self.hostname = 'localhost'
context_factory = ssl_context_factory(self.key_file, self.certificate_file)
context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file))
server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname)
self.server = yield server_endpoint.listen(self.site)

14
tests/test_pipeline_crawl.py
View File

@ -1,5 +1,5 @@
import os
import shutil
from pathlib import Path
from testfixtures import LogCapture
from twisted.internet import defer
@ -61,12 +61,12 @@ class FileDownloadCrawlTestCase(TestCase):
self.mockserver.__enter__()
# prepare a directory for storing files
self.tmpmediastore = self.mktemp()
os.mkdir(self.tmpmediastore)
self.tmpmediastore = Path(self.mktemp())
self.tmpmediastore.mkdir()
self.settings = {
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'ITEM_PIPELINES': {self.pipeline_class: 1},
self.store_setting_key: self.tmpmediastore,
self.store_setting_key: str(self.tmpmediastore),
}
self.runner = CrawlerRunner(self.settings)
self.items = []
@ -111,9 +111,7 @@ class FileDownloadCrawlTestCase(TestCase):
# check that the image files where actually written to the media store
for item in items:
for i in item[self.media_key]:
self.assertTrue(
os.path.exists(
os.path.join(self.tmpmediastore, i['path'])))
self.assertTrue((self.tmpmediastore / i['path']).exists())
def _assert_files_download_failure(self, crawler, items, code, logs):
@ -133,7 +131,7 @@ class FileDownloadCrawlTestCase(TestCase):
self.assertEqual(logs.count(file_dl_failure), 3)
# check that no files were written to the media store
self.assertEqual(os.listdir(self.tmpmediastore), [])
self.assertEqual([x for x in self.tmpmediastore.iterdir()], [])
@defer.inlineCallbacks
def test_download_media(self):

3
tests/test_pipeline_files.py
View File

@ -3,6 +3,7 @@ import random
import time
from datetime import datetime
from io import BytesIO
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
from unittest import mock
@ -89,7 +90,7 @@ class FilesPipelineTestCase(unittest.TestCase):
self.assertEqual(self.pipeline.store.basedir, self.tempdir)
path = 'some/image/key.jpg'
fullpath = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg')
self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath)
@defer.inlineCallbacks

6
tests/test_proxy_connect.py
View File

@ -2,6 +2,7 @@ import json
import os
import re
import sys
from pathlib import Path
from subprocess import Popen, PIPE
from urllib.parse import urlsplit, urlunsplit
from testfixtures import LogCapture
@ -27,14 +28,13 @@ from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'keys', 'mitmproxy-ca.pem')
cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem'
self.proc = Popen([sys.executable,
'-c', script,
'--listen-host', '127.0.0.1',
'--listen-port', '0',
'--proxyauth', f'{self.auth_user}:{self.auth_pass}',
'--certs', cert_path,
'--certs', str(cert_path),
'--ssl-insecure',
],
stdout=PIPE, env=get_testenv())

40
tests/test_spiderloader/__init__.py
View File

@ -1,7 +1,7 @@
import sys
import os
import shutil
import warnings
from pathlib import Path
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@ -17,10 +17,10 @@ from scrapy.settings import Settings
from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
module_dir = os.path.dirname(os.path.abspath(__file__))
module_dir = Path(__file__).resolve().parent
def _copytree(source, target):
def _copytree(source: Path, target: Path):
try:
shutil.copytree(source, target)
except shutil.Error:
@ -30,18 +30,18 @@ def _copytree(source, target):
class SpiderLoaderTest(unittest.TestCase):
def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
self.tmpdir = tempfile.mkdtemp()
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
orig_spiders_dir = module_dir / 'test_spiders'
self.tmpdir = Path(tempfile.mkdtemp())
self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
_copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir)
sys.path.append(str(self.tmpdir))
settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
self.spider_loader = SpiderLoader.from_settings(settings)
def tearDown(self):
del self.spider_loader
del sys.modules['test_spiders_xxx']
sys.path.remove(self.tmpdir)
sys.path.remove(str(self.tmpdir))
def test_interface(self):
verifyObject(ISpiderLoader, self.spider_loader)
@ -135,22 +135,22 @@ class SpiderLoaderTest(unittest.TestCase):
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def setUp(self):
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
self.tmpdir = self.mktemp()
os.mkdir(self.tmpdir)
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
orig_spiders_dir = module_dir / 'test_spiders'
self.tmpdir = Path(self.mktemp())
self.tmpdir.mkdir()
self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
_copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir)
sys.path.append(str(self.tmpdir))
self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
def tearDown(self):
del sys.modules['test_spiders_xxx']
sys.path.remove(self.tmpdir)
sys.path.remove(str(self.tmpdir))
def test_dupename_warning(self):
# copy 1 spider module so as to have duplicate spider name
shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'),
os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py'))
shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py',
self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py')
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
@ -171,10 +171,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def test_multiple_dupename_warning(self):
# copy 2 spider modules so as to have duplicate spider name
# This should issue 2 warning, 1 for each duplicate spider name
shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'),
os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py'))
shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'),
os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py'))
shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py',
self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py')
shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py',
self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py')
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)

4
tests/test_spiderstate.py
View File

@ -1,5 +1,5 @@
import os
from datetime import datetime
from pathlib import Path
import shutil
from twisted.trial import unittest
@ -13,7 +13,7 @@ class SpiderStateTest(unittest.TestCase):
def test_store_load(self):
jobdir = self.mktemp()
os.mkdir(jobdir)
Path(jobdir).mkdir()
try:
spider = Spider(name='default')
dt = datetime.now()

44
tests/test_utils_gz.py
View File

@ -1,5 +1,5 @@
import unittest
from os.path import join
from pathlib import Path
from w3lib.encoding import html_to_unicode
@ -8,46 +8,40 @@ from scrapy.http import Response
from tests import tests_datadir
SAMPLEDIR = join(tests_datadir, 'compressed')
SAMPLEDIR = Path(tests_datadir, 'compressed')
class GunzipTest(unittest.TestCase):
def test_gunzip_basic(self):
with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:
r1 = Response("http://www.example.com", body=f.read())
self.assertTrue(gzip_magic_number(r1))
r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes())
self.assertTrue(gzip_magic_number(r1))
r2 = Response("http://www.example.com", body=gunzip(r1.body))
self.assertFalse(gzip_magic_number(r2))
self.assertEqual(len(r2.body), 9950)
r2 = Response("http://www.example.com", body=gunzip(r1.body))
self.assertFalse(gzip_magic_number(r2))
self.assertEqual(len(r2.body), 9950)
def test_gunzip_truncated(self):
with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:
text = gunzip(f.read())
assert text.endswith(b'</html')
text = gunzip((SAMPLEDIR / 'truncated-crc-error.gz').read_bytes())
assert text.endswith(b'</html')
def test_gunzip_no_gzip_file_raises(self):
with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f:
self.assertRaises(IOError, gunzip, f.read())
self.assertRaises(IOError, gunzip, (SAMPLEDIR / 'feed-sample1.xml').read_bytes())
def test_gunzip_truncated_short(self):
with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
r1 = Response("http://www.example.com", body=f.read())
self.assertTrue(gzip_magic_number(r1))
r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'truncated-crc-error-short.gz').read_bytes())
self.assertTrue(gzip_magic_number(r1))
r2 = Response("http://www.example.com", body=gunzip(r1.body))
assert r2.body.endswith(b'</html>')
self.assertFalse(gzip_magic_number(r2))
r2 = Response("http://www.example.com", body=gunzip(r1.body))
assert r2.body.endswith(b'</html>')
self.assertFalse(gzip_magic_number(r2))
def test_is_gzipped_empty(self):
r1 = Response("http://www.example.com")
self.assertFalse(gzip_magic_number(r1))
def test_gunzip_illegal_eof(self):
with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f:
text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1]
with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o:
expected_text = o.read().decode("utf-8")
self.assertEqual(len(text), len(expected_text))
self.assertEqual(text, expected_text)
text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1]
expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8")
self.assertEqual(len(text), len(expected_text))
self.assertEqual(text, expected_text)

7
tests/test_utils_iterators.py
View File

@ -1,5 +1,3 @@
import os
from pytest import mark
from twisted.trial import unittest
@ -303,11 +301,6 @@ class LxmlXmliterTestCase(XmliterTestCase):
class UtilsCsvTestCase(unittest.TestCase):
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')
sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv')
sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv')
def test_csviter_defaults(self):
body = get_testdata('feeds', 'feed-sample3.csv')
response = TextResponse(url="http://example.com/", body=body)

3
tests/test_utils_misc/__init__.py
View File

@ -1,6 +1,7 @@
import sys
import os
import unittest
from pathlib import Path
from unittest import mock
from scrapy.item import Item, Field
@ -55,7 +56,7 @@ class UtilsMiscTestCase(unittest.TestCase):
self.assertRaises(ImportError, walk_modules, 'nomodule999')
def test_walk_modules_egg(self):
egg = os.path.join(os.path.dirname(__file__), 'test.egg')
egg = str(Path(__file__).parent / 'test.egg')
sys.path.append(egg)
try:
mods = walk_modules('testegg')

17
tests/test_utils_project.py
View File

@ -4,6 +4,7 @@ import tempfile
import shutil
import contextlib
import warnings
from pathlib import Path
from pytest import warns
@ -18,9 +19,7 @@ def inside_a_project():
try:
os.chdir(project_dir)
with open('scrapy.cfg', 'w') as f:
# create an empty scrapy.cfg
f.close()
Path('scrapy.cfg').touch()
yield project_dir
finally:
@ -31,20 +30,20 @@ def inside_a_project():
class ProjectUtilsTest(unittest.TestCase):
def test_data_path_outside_project(self):
self.assertEqual(
os.path.join('.scrapy', 'somepath'),
str(Path('.scrapy', 'somepath')),
data_path('somepath')
)
abspath = os.path.join(os.path.sep, 'absolute', 'path')
abspath = str(Path(os.path.sep, 'absolute', 'path'))
self.assertEqual(abspath, data_path(abspath))
def test_data_path_inside_project(self):
with inside_a_project() as proj_path:
expected = os.path.join(proj_path, '.scrapy', 'somepath')
expected = Path(proj_path, '.scrapy', 'somepath')
self.assertEqual(
os.path.realpath(expected),
os.path.realpath(data_path('somepath'))
expected.resolve(),
Path(data_path('somepath')).resolve()
)
abspath = os.path.join(os.path.sep, 'absolute', 'path')
abspath = str(Path(os.path.sep, 'absolute', 'path').resolve())
self.assertEqual(abspath, data_path(abspath))

12
tests/test_utils_response.py
View File

@ -1,6 +1,6 @@
import os
import unittest
import warnings
from pathlib import Path
from urllib.parse import urlparse
from scrapy.exceptions import ScrapyDeprecationWarning
@ -39,10 +39,9 @@ class ResponseUtilsTest(unittest.TestCase):
def browser_open(burl):
path = urlparse(burl).path
if not os.path.exists(path):
if not path or not Path(path).exists():
path = burl.replace('file://', '')
with open(path, "rb") as f:
bbody = f.read()
bbody = Path(path).read_bytes()
self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
return True
response = HtmlResponse(url, body=body)
@ -98,10 +97,9 @@ class ResponseUtilsTest(unittest.TestCase):
def check_base_url(burl):
path = urlparse(burl).path
if not os.path.exists(path):
if not path or not Path(path).exists():
path = burl.replace('file://', '')
with open(path, "rb") as f:
bbody = f.read()
bbody = Path(path).read_bytes()
self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
return True

22
tests/test_utils_template.py
View File

@ -1,4 +1,4 @@
import os
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
import unittest
@ -22,21 +22,19 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
template = 'from ${project_name}.spiders.${name} import ${classname}'
rendered = 'from proj.spiders.spi import TheSpider'
template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
render_path = os.path.join(self.tmp_path, 'templ.py')
template_path = Path(self.tmp_path, 'templ.py.tmpl')
render_path = Path(self.tmp_path, 'templ.py')
with open(template_path, 'wb') as tmpl_file:
tmpl_file.write(template.encode('utf8'))
assert os.path.isfile(template_path) # Failure of test itself
template_path.write_text(template, encoding='utf8')
assert template_path.is_file() # Failure of test itself
render_templatefile(template_path, **context)
render_templatefile(str(template_path), **context)
self.assertFalse(os.path.exists(template_path))
with open(render_path, 'rb') as result:
self.assertEqual(result.read().decode('utf8'), rendered)
self.assertFalse(template_path.exists())
self.assertEqual(render_path.read_text(encoding='utf8'), rendered)
os.remove(render_path)
assert not os.path.exists(render_path) # Failure of test itself
render_path.unlink()
assert not render_path.exists() # Failure of test itself
if '__main__' == __name__:

19
tests/test_webclient.py
View File

@ -2,8 +2,8 @@
from twisted.internet import defer
Tests borrowed from the twisted.web.client tests.
"""
import os
import shutil
from pathlib import Path
import OpenSSL.SSL
from twisted.trial import unittest
@ -15,7 +15,6 @@ except ImportError:
# deprecated in Twisted 19.7.0
# (remove once we bump our requirement past that version)
from twisted.test.proto_helpers import StringTransport
from twisted.python.filepath import FilePath
from twisted.protocols.policies import WrappingFactory
from twisted.internet.defer import inlineCallbacks
@ -230,10 +229,10 @@ class WebClientTestCase(unittest.TestCase):
return reactor.listenTCP(0, site, interface="127.0.0.1")
def setUp(self):
self.tmpname = self.mktemp()
os.mkdir(self.tmpname)
FilePath(self.tmpname).child("file").setContent(b"0123456789")
r = static.File(self.tmpname)
self.tmpname = Path(self.mktemp())
self.tmpname.mkdir()
(self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(str(self.tmpname))
r.putChild(b"redirect", util.Redirect(b"/file"))
r.putChild(b"wait", ForeverTakingResource())
r.putChild(b"error", ErrorResource())
@ -379,10 +378,10 @@ class WebClientSSLTestCase(unittest.TestCase):
return f"https://127.0.0.1:{self.portno}/{path}"
def setUp(self):
self.tmpname = self.mktemp()
os.mkdir(self.tmpname)
FilePath(self.tmpname).child("file").setContent(b"0123456789")
r = static.File(self.tmpname)
self.tmpname = Path(self.mktemp())
self.tmpname.mkdir()
(self.tmpname / "file").write_bytes(b"0123456789")
r = static.File(str(self.tmpname))
r.putChild(b"payload", PayloadResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)