From 6beb4f0119619170edfb4579f28da22c9250e84b Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Mon, 23 May 2016 23:14:33 -0300 Subject: [PATCH 01/10] Created project_dir optional parameter in startproject command line --- scrapy/commands/startproject.py | 66 +++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index b3c5011df..8f3243a2a 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,9 +1,10 @@ from __future__ import print_function import re +import os import string from importlib import import_module from os.path import join, exists, abspath -from shutil import copytree, ignore_patterns, move +from shutil import ignore_patterns, move, copy2, copystat import scrapy from scrapy.commands import ScrapyCommand @@ -27,7 +28,7 @@ class Command(ScrapyCommand): default_settings = {'LOG_ENABLED': False} def syntax(self): - return "" + return " [project_dir]" def short_desc(self): return "Create new project" @@ -51,20 +52,71 @@ class Command(ScrapyCommand): return True return False + def _copytree(self, src, dst, symlinks=False, ignore=None): + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + if not os.path.exists(dst): + os.makedirs(dst) + + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if symlinks and os.path.islink(srcname): + linkto = os.readlink(srcname) + os.symlink(linkto, dstname) + elif os.path.isdir(srcname): + self._copytree(srcname, dstname, symlinks, ignore) + else: + # Will raise a SpecialFileError for unsupported file types + copy2(srcname, dstname) + # catch the Error from the recursive copytree so that we can + # continue with other files + except EnvironmentError, err: + errors.extend(err.args[0]) + except EnvironmentError, why: + errors.append((srcname, dstname, str(why))) + try: + copystat(src, dst) + except OSError, why: + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.append((src, dst, str(why))) + if errors: + raise EnvironmentError(errors) + def run(self, args, opts): - if len(args) != 1: + if len(args) not in (1, 2): raise UsageError() + project_name = args[0] + project_dir = args[0] + + if len(args) == 2: + project_dir = args[1] + if exists(join(project_dir, 'scrapy.cfg')): + self.exitcode = 1 + print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) + return if not self._is_valid_name(project_name): self.exitcode = 1 return - copytree(self.templates_dir, project_name, ignore=IGNORE) - move(join(project_name, 'module'), join(project_name, project_name)) + self._copytree(self.templates_dir, abspath(project_dir), ignore=IGNORE) + move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) - tplfile = join(project_name, + tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) @@ -80,4 +132,4 @@ class Command(ScrapyCommand): _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ join(scrapy.__path__[0], 'templates') return join(_templates_base_dir, 'project') - \ No newline at end of file + From 2521f031d6db5c986ab88dc37c793513f771c3b1 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Mon, 23 May 2016 23:15:53 -0300 Subject: [PATCH 02/10] Created new tests for implementation --- tests/test_commands.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2e47160d7..42cade849 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -73,6 +73,23 @@ class StartprojectTest(ProjectTest): self.assertEqual(1, self.call('startproject', 'wrong---project---name')) self.assertEqual(1, self.call('startproject', 'sys')) + def test_startproject_with_project_dir(self): + project_dir = mkdtemp() + self.assertEqual(0, self.call('startproject', self.project_name, project_dir)) + + assert exists(join(abspath(project_dir), 'scrapy.cfg')) + assert exists(join(abspath(project_dir), 'testproject')) + assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) + + self.assertEqual(1, self.call('startproject', self.project_name, project_dir)) + self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) + self.assertEqual(1, self.call('startproject', 'wrong---project---name')) + self.assertEqual(1, self.call('startproject', 'sys')) + class StartprojectTemplatesTest(ProjectTest): From 089483aece181913bf19548725653462a52febdf Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Mon, 23 May 2016 23:16:15 -0300 Subject: [PATCH 03/10] Updated docs for new option in command line --- docs/topics/commands.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 9a40a2c29..8e1a0ad0e 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -103,13 +103,14 @@ Creating projects The first thing you typically do with the ``scrapy`` tool is create your Scrapy project:: - scrapy startproject myproject + scrapy startproject myproject [project_dir] -That will create a Scrapy project under the ``myproject`` directory. +That will create a Scrapy project under the ``project_dir`` directory. +If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``. Next, you go inside the new project directory:: - cd myproject + cd project_dir And you're ready to use the ``scrapy`` command to manage and control your project from there. @@ -181,11 +182,12 @@ Project-only commands: startproject ------------ -* Syntax: ``scrapy startproject `` +* Syntax: ``scrapy startproject [project_dir]`` * Requires project: *no* -Creates a new Scrapy project named ``project_name``, under the ``project_name`` +Creates a new Scrapy project named ``project_name``, under the ``project_dir`` directory. +If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``. Usage example:: From 24a45cc6e6d15eab7aafebd4cde5364201d2b133 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 24 May 2016 10:58:50 -0300 Subject: [PATCH 04/10] Fix py35 compatibility tests --- scrapy/commands/startproject.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 8f3243a2a..4da129865 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -79,13 +79,13 @@ class Command(ScrapyCommand): copy2(srcname, dstname) # catch the Error from the recursive copytree so that we can # continue with other files - except EnvironmentError, err: + except EnvironmentError as err: errors.extend(err.args[0]) - except EnvironmentError, why: + except EnvironmentError as why: errors.append((srcname, dstname, str(why))) try: copystat(src, dst) - except OSError, why: + except OSError as why: if WindowsError is not None and isinstance(why, WindowsError): # Copying file access times may fail on Windows pass From fc9a45ee9129912e4ee138b0a17d8ed8a27a0c9e Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 24 May 2016 11:57:56 -0300 Subject: [PATCH 05/10] Simplified copytree function --- scrapy/commands/startproject.py | 40 ++++++++------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 4da129865..c0e7af88a 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -52,47 +52,25 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src, dst, symlinks=False, ignore=None): + def _copytree(self, src, dst): + ignore = IGNORE names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() + ignored_names = ignore(src, names) if not os.path.exists(dst): os.makedirs(dst) - errors = [] for name in names: if name in ignored_names: continue + srcname = os.path.join(src, name) dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - self._copytree(srcname, dstname, symlinks, ignore) - else: - # Will raise a SpecialFileError for unsupported file types - copy2(srcname, dstname) - # catch the Error from the recursive copytree so that we can - # continue with other files - except EnvironmentError as err: - errors.extend(err.args[0]) - except EnvironmentError as why: - errors.append((srcname, dstname, str(why))) - try: - copystat(src, dst) - except OSError as why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass + if os.path.isdir(srcname): + self._copytree(srcname, dstname) else: - errors.append((src, dst, str(why))) - if errors: - raise EnvironmentError(errors) + copy2(srcname, dstname) + copystat(src, dst) def run(self, args, opts): if len(args) not in (1, 2): @@ -112,7 +90,7 @@ class Command(ScrapyCommand): self.exitcode = 1 return - self._copytree(self.templates_dir, abspath(project_dir), ignore=IGNORE) + self._copytree(self.templates_dir, abspath(project_dir)) move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) From b8a09d7ab7baad9e8ea2e7a8dd1ed0fe4e4fedc7 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 24 May 2016 11:58:52 -0300 Subject: [PATCH 06/10] Added tests for more or less parameters --- tests/test_commands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 42cade849..a487d14f7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -89,6 +89,8 @@ class StartprojectTest(ProjectTest): self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) self.assertEqual(1, self.call('startproject', 'wrong---project---name')) self.assertEqual(1, self.call('startproject', 'sys')) + self.assertEqual(2, self.call('startproject')) + self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) class StartprojectTemplatesTest(ProjectTest): From 85c4ecb92856f7d50f8dbf7c92d629bd5db83f02 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 24 May 2016 13:00:41 -0300 Subject: [PATCH 07/10] Removed validation of project_name dir exists --- scrapy/commands/startproject.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index c0e7af88a..a51a586ab 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -44,8 +44,6 @@ class Command(ScrapyCommand): if not re.search(r'^[_a-zA-Z]\w*$', project_name): print('Error: Project names must begin with a letter and contain'\ ' only\nletters, numbers and underscores') - elif exists(project_name): - print('Error: Directory %r already exists' % project_name) elif _module_exists(project_name): print('Error: Module %r already exists' % project_name) else: From 9ad54b381535eec583e387cb75d7205ccc8d7d3a Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 24 May 2016 13:03:33 -0300 Subject: [PATCH 08/10] Fix template description after create project --- scrapy/commands/startproject.py | 4 ++-- tests/test_commands.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index a51a586ab..0940d67f9 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -98,9 +98,9 @@ class Command(ScrapyCommand): ProjectName=string_camelcase(project_name)) print("New Scrapy project %r, using template directory %r, created in:" % \ (project_name, self.templates_dir)) - print(" %s\n" % abspath(project_name)) + print(" %s\n" % abspath(project_dir)) print("You can start your first spider with:") - print(" cd %s" % project_name) + print(" cd %s" % project_dir) print(" scrapy genspider example example.com") @property diff --git a/tests/test_commands.py b/tests/test_commands.py index a487d14f7..ca7a19eaa 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -85,6 +85,8 @@ class StartprojectTest(ProjectTest): assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) + self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) + self.assertEqual(1, self.call('startproject', self.project_name, project_dir)) self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) self.assertEqual(1, self.call('startproject', 'wrong---project---name')) From de64a1f68afe24a246152f778b7f3b236b9ae439 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 19 Jul 2016 00:04:45 -0300 Subject: [PATCH 09/10] Fix scrapy.cfg validation Signed-off-by: Felipe Ruhland --- scrapy/commands/startproject.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 0940d67f9..2218bd134 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -79,10 +79,11 @@ class Command(ScrapyCommand): if len(args) == 2: project_dir = args[1] - if exists(join(project_dir, 'scrapy.cfg')): - self.exitcode = 1 - print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) - return + + if exists(join(project_dir, 'scrapy.cfg')): + self.exitcode = 1 + print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) + return if not self._is_valid_name(project_name): self.exitcode = 1 From fe088925a3eff9d1de5682efaa33a391c6dc7744 Mon Sep 17 00:00:00 2001 From: Felipe Ruhland Date: Tue, 19 Jul 2016 00:12:39 -0300 Subject: [PATCH 10/10] Included implementation notes in docstring Signed-off-by: Felipe Ruhland --- scrapy/commands/startproject.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 2218bd134..e3989baaf 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -51,6 +51,14 @@ class Command(ScrapyCommand): return False def _copytree(self, src, dst): + """ + Since the original function always creates the directory, to resolve + the issue a new function had to be created. It's a simple copy and + was reduced for this case. + + More info at: + https://github.com/scrapy/scrapy/pull/2005 + """ ignore = IGNORE names = os.listdir(src) ignored_names = ignore(src, names)