From 4ff979428612d8a2443996a36872631434f04487 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 26 Dec 2018 02:15:44 -0400 Subject: [PATCH] Update and move JavaScript install code Update the JavaScript dependency installation code to handle scoped packages. The code is also updated to use pathlib's Path. Move the JavaScript dependency installation to its own app named dependencies. Signed-off-by: Roberto Rosario --- mayan/apps/common/classes.py | 1 - mayan/apps/common/exceptions.py | 8 - mayan/apps/common/javascript.py | 194 ------------------ .../management/commands/initialsetup.py | 8 - .../management/commands/performupgrade.py | 2 - mayan/apps/dependencies/__init__.py | 3 + mayan/apps/dependencies/apps.py | 29 +++ mayan/apps/dependencies/exceptions.py | 7 + mayan/apps/dependencies/handlers.py | 8 + mayan/apps/dependencies/javascript.py | 188 +++++++++++++++++ .../apps/dependencies/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/installjavascript.py | 0 mayan/settings/base.py | 1 + 14 files changed, 236 insertions(+), 213 deletions(-) delete mode 100644 mayan/apps/common/javascript.py create mode 100644 mayan/apps/dependencies/__init__.py create mode 100644 mayan/apps/dependencies/apps.py create mode 100644 mayan/apps/dependencies/exceptions.py create mode 100644 mayan/apps/dependencies/handlers.py create mode 100644 mayan/apps/dependencies/javascript.py create mode 100644 mayan/apps/dependencies/management/__init__.py create mode 100644 mayan/apps/dependencies/management/commands/__init__.py rename mayan/apps/{common => dependencies}/management/commands/installjavascript.py (100%) diff --git a/mayan/apps/common/classes.py b/mayan/apps/common/classes.py index 2c52f2e16a..85de871f00 100644 --- a/mayan/apps/common/classes.py +++ b/mayan/apps/common/classes.py @@ -281,7 +281,6 @@ class Template(object): result.append(template.render(request=request)) return result - @classmethod def get(cls, name): return cls._registry[name] diff --git a/mayan/apps/common/exceptions.py b/mayan/apps/common/exceptions.py index e74b95ecf5..2576dc7435 100644 --- a/mayan/apps/common/exceptions.py +++ b/mayan/apps/common/exceptions.py @@ -42,11 +42,3 @@ class UnknownLatestVersion(BaseCommonException): """ It is not possible to determine what is the latest upstream version. """ - - -class NPMException(BaseCommonException): - """Base exception for the NPM registry client""" - - -class NPMPackgeIntegrityError(NPMException): - """Hash mismatch exception""" diff --git a/mayan/apps/common/javascript.py b/mayan/apps/common/javascript.py deleted file mode 100644 index eb9305835b..0000000000 --- a/mayan/apps/common/javascript.py +++ /dev/null @@ -1,194 +0,0 @@ -from __future__ import print_function, unicode_literals - -import base64 -import hashlib -import json -import os -import shutil -import tarfile - -from furl import furl -from pathlib2 import Path -import requests -from semver import max_satisfying - -from django.apps import apps -from django.utils.encoding import force_bytes - -from .exceptions import NPMException, NPMPackgeIntegrityError -from .utils import mkdtemp - - -class NPMPackage(object): - def __init__(self, registry, name, version): - self.registry = registry - self.name = name - self.version = version - - def _download(self): - with requests.get(self.download_metadata['dist']['tarball'], stream=True) as response: - with open(name=self.tar_file_path, mode='wb') as file_object: - file_object.write(response.content) - - try: - upstream_algorithm_name, upstream_integrity_value = self.download_metadata['dist']['integrity'].split('-', 1) - except KeyError: - upstream_algorithm_name = 'sha1' - upstream_integrity_value = self.download_metadata['dist']['shasum'] - - algorithms = { - 'sha1': lambda data: hashlib.sha1(data).hexdigest(), - 'sha256': lambda data: base64.b64encode(hashlib.sha256(data).digest()), - 'sha512': lambda data: base64.b64encode(hashlib.sha512(data).digest()), - } - - try: - algorithm = algorithms[upstream_algorithm_name] - except KeyError: - raise NPMException( - 'Unknown hash algorithm: {}'.format(upstream_algorithm_name) - ) - - with open(name=self.tar_file_path, mode='rb') as file_object: - integrity_value = algorithm(file_object.read()) - - if integrity_value != upstream_integrity_value: - os.unlink(self.tar_file_path) - raise NPMPackgeIntegrityError( - 'Hash of downloaded package doesn\'t match online version.' - ) - - def _extract(self): - shutil.rmtree( - os.path.join( - self.registry.module_directory, self.name - ), ignore_errors=True - ) - - with tarfile.open(name=self.tar_file_path, mode='r') as file_object: - file_object.extractall(path=self.registry.module_directory) - - os.rename( - os.path.join(self.registry.module_directory, 'package'), - os.path.join(self.registry.module_directory, self.name) - ) - - @property - def best_version(self): - if not hasattr(self, '_best_version'): - self._best_version = max_satisfying( - self.versions, force_bytes(self.version), loose=True - ) - print('Best version: {}'.format(self._best_version)) - - return self._best_version - - @property - def download_metadata(self): - if not hasattr(self, '_download_metadata'): - response = requests.get(url=self.download_url) - self._download_metadata = response.json() - return self._download_metadata - - @property - def download_url(self): - f = furl(self.url) - f.path.segments = f.path.segments + [self.best_version] - return f.tostr() - - def install(self, include_dependencies=False): - print('Installing package: {}{}'.format(self.name, self.version)) - - self._download() - self._extract() - - if include_dependencies: - for name, version in self.download_metadata.get('dependencies', {}).items(): - package = NPMPackage( - registry=self.registry, name=name, version=version - ) - package.install() - - @property - def tar_filename(self): - if not hasattr(self, '_tar_filename'): - self._tar_filename = furl( - self.download_metadata['dist']['tarball'] - ).path.segments[-1] - - return self._tar_filename - - @property - def tar_file_path(self): - if not hasattr(self, '_tar_file_path'): - self._tar_file_path = os.path.join( - self.registry.cache_path, self.tar_filename - ) - - return self._tar_file_path - - @property - def url(self): - f = furl(self.registry.url) - f.path.segments = f.path.segments + [self.name] - return f.tostr() - - @property - def versions(self): - if not hasattr(self, '_versions'): - response = requests.get(url=self.url) - self._versions = [ - force_bytes(version) for version in response.json()['versions'].keys() - ] - - return self._versions - - -class NPMRegistry(object): - DEFAULT_REGISTRY_URL = 'http://registry.npmjs.com' - DEFAULT_MODULE_DIRECTORY = 'node_modules' - DEFAULT_PACKAGE_FILENAME = 'package.json' - DEFAULT_LOCK_FILENAME = 'package-lock.json' - - def __init__(self, url=None, cache_path=None, module_directory=None, package_filename=None, lock_filename=None): - self.url = url or self.DEFAULT_REGISTRY_URL - self.cache_path = cache_path or mkdtemp() - self.module_directory = module_directory or self.DEFAULT_MODULE_DIRECTORY - self.package_file = package_filename or self.DEFAULT_PACKAGE_FILENAME - self.lock_filename = lock_filename or self.DEFAULT_LOCK_FILENAME - - def _install_package(self, name, version): - package = NPMPackage(registry=self, name=name, version=version) - package.install() - - def _read_package(self): - with open(self.package_file) as file_object: - self._package_data = json.loads(file_object.read()) - - def install(self, package=None): - if package: - name, version = package.split('@') - self._install_package(name=name, version=version) - else: - self._read_package() - - for name, version in self._package_data['dependencies'].items(): - self._install_package(name=name, version=version) - - -class JSDependencyManager(object): - def install(self, app_name=None): - if app_name: - app_config_list = [apps.get_app_config(app_label=app_name)] - else: - app_config_list = apps.get_app_configs() - - for app in app_config_list: - for root, dirs, files in os.walk(os.path.join(app.path, 'static')): - if 'package.json' in files and not (set(Path(root).parts) & set(['node_modules', 'packages', 'vendors'])): - print('Installing JavaScript packages for app: {} - {}'.format(app.label, root)) - npm_client = NPMRegistry( - module_directory=os.path.join(root, 'node_modules'), - package_filename=os.path.join(root, 'package.json') - ) - npm_client.install() diff --git a/mayan/apps/common/management/commands/initialsetup.py b/mayan/apps/common/management/commands/initialsetup.py index 456983279c..24647167e7 100644 --- a/mayan/apps/common/management/commands/initialsetup.py +++ b/mayan/apps/common/management/commands/initialsetup.py @@ -27,11 +27,6 @@ class Command(management.BaseCommand): help='Force execution of the initialization process.', ) - parser.add_argument( - '--no-javascript', action='store_true', dest='no_javascript', - help='Don\'t download the JavaScript dependencies.', - ) - def initialize_system(self, force=False): system_path = os.path.join(settings.MEDIA_ROOT, SYSTEM_DIR) settings_path = os.path.join(settings.MEDIA_ROOT, 'mayan_settings') @@ -88,8 +83,5 @@ class Command(management.BaseCommand): self.initialize_system(force=options.get('force', False)) pre_initial_setup.send(sender=self) - if not options.get('no_javascript', False): - management.call_command('installjavascript', interactive=False) - management.call_command('createautoadmin', interactive=False) post_initial_setup.send(sender=self) diff --git a/mayan/apps/common/management/commands/performupgrade.py b/mayan/apps/common/management/commands/performupgrade.py index d2ba4999c8..9e94e7360b 100644 --- a/mayan/apps/common/management/commands/performupgrade.py +++ b/mayan/apps/common/management/commands/performupgrade.py @@ -19,8 +19,6 @@ class Command(management.BaseCommand): ) ) - management.call_command('installjavascript', interactive=False) - try: perform_upgrade.send(sender=self) except Exception as exception: diff --git a/mayan/apps/dependencies/__init__.py b/mayan/apps/dependencies/__init__.py new file mode 100644 index 0000000000..790c8ccc9f --- /dev/null +++ b/mayan/apps/dependencies/__init__.py @@ -0,0 +1,3 @@ +from __future__ import unicode_literals + +default_app_config = 'mayan.apps.dependencies.apps.DependenciesApp' diff --git a/mayan/apps/dependencies/apps.py b/mayan/apps/dependencies/apps.py new file mode 100644 index 0000000000..0f6850a5a5 --- /dev/null +++ b/mayan/apps/dependencies/apps.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common import MayanAppConfig +from mayan.apps.common.signals import post_initial_setup, post_upgrade + +from .handlers import handler_install_javascript + + +class DependenciesApp(MayanAppConfig): + app_namespace = 'dependencies' + app_url = 'dependencies' + has_rest_api = False + has_tests = False + name = 'mayan.apps.dependencies' + verbose_name = _('Dependencies') + + def ready(self): + super(DependenciesApp, self).ready() + + post_initial_setup.connect( + dispatch_uid='dependendies_handler_install_javascript', + receiver=handler_install_javascript + ) + post_upgrade.connect( + dispatch_uid='dependendies_handler_install_javascript', + receiver=handler_install_javascript + ) diff --git a/mayan/apps/dependencies/exceptions.py b/mayan/apps/dependencies/exceptions.py new file mode 100644 index 0000000000..4a7799a31e --- /dev/null +++ b/mayan/apps/dependencies/exceptions.py @@ -0,0 +1,7 @@ +from __future__ import unicode_literals + + +class DependenciesException(Exception): + """ + Base exception for the dependencies app + """ diff --git a/mayan/apps/dependencies/handlers.py b/mayan/apps/dependencies/handlers.py new file mode 100644 index 0000000000..00ccc8ff42 --- /dev/null +++ b/mayan/apps/dependencies/handlers.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +from .javascript import JSDependencyManager + + +def handler_install_javascript(sender, **kwargs): + js_manager = JSDependencyManager() + js_manager.install() diff --git a/mayan/apps/dependencies/javascript.py b/mayan/apps/dependencies/javascript.py new file mode 100644 index 0000000000..6f6806d7b8 --- /dev/null +++ b/mayan/apps/dependencies/javascript.py @@ -0,0 +1,188 @@ +from __future__ import print_function, unicode_literals + +import base64 +import hashlib +import json +import shutil +import tarfile + +from furl import furl +from pathlib2 import Path +import requests +from semver import max_satisfying + +from django.apps import apps +from django.utils.encoding import force_bytes, force_text +from django.utils.functional import cached_property + +from mayan.apps.common.utils import mkdtemp + +from .exceptions import DependenciesException + +DEFAULT_REGISTRY_URL = 'http://registry.npmjs.com' +DEFAULT_MODULE_DIRECTORY = 'node_modules' +DEFAULT_PACKAGE_FILENAME = 'package.json' +DEFAULT_LOCK_FILENAME = 'package-lock.json' + + +class NPMPackage(object): + def __init__(self, registry, name, version): + self.registry = registry + self.name = name + self.version = version + + def download(self): + algorithm_function = self.get_algorithm_function() + tar_file_path = self.get_tar_file_path() + + with requests.get(self.version_metadata['dist']['tarball'], stream=True) as response: + with tar_file_path.open(mode='wb') as file_object: + shutil.copyfileobj(response.raw, file_object) + + with tar_file_path.open(mode='rb') as file_object: + integrity_is_good = algorithm_function(file_object.read()) + + if not integrity_is_good: + tar_file_path.unlink() + raise DependenciesException( + 'Hash of downloaded package doesn\'t match online version.' + ) + + def extract(self): + shutil.rmtree( + path=force_text( + Path( + self.registry.module_directory, self.name + ) + ), ignore_errors=True + ) + + with tarfile.open(name=force_text(self.get_tar_file_path()), mode='r') as file_object: + file_object.extractall( + path=force_text(self.registry.module_directory) + ) + + Path(self.registry.module_directory, 'package').rename( + target=Path(self.registry.module_directory, self.name) + ) + + def get_algorithm_function(self): + try: + integrity = self.version_metadata['dist']['integrity'] + except KeyError: + algorithm_name = 'sha1' + integrity_value = self.version_metadata['dist']['shasum'] + else: + algorithm_name, integrity_value = integrity.split('-', 1) + + algorithms = { + 'sha1': lambda data: hashlib.sha1(data).hexdigest() == integrity_value, + 'sha256': lambda data: base64.b64encode(hashlib.sha256(data).digest()) == integrity_value, + 'sha512': lambda data: base64.b64encode(hashlib.sha512(data).digest()) == integrity_value, + } + + try: + algorithm = algorithms[algorithm_name] + except KeyError: + raise DependenciesException( + 'Unknown hash algorithm: {}'.format(algorithm_name) + ) + else: + return algorithm + + def get_best_version(self): + return max_satisfying( + self.versions, force_bytes(self.version), loose=True + ) + + def get_tar_file_path(self): + return Path( + self.registry.cache_path, self.get_tar_filename() + ) + + def get_tar_filename(self): + return furl( + self.version_metadata['dist']['tarball'] + ).path.segments[-1] + + def install(self, include_dependencies=False): + print('Installing package: {}{}'.format(self.name, self.version)) + + self.download() + self.extract() + + if include_dependencies: + for name, version in self.version_metadata.get('dependencies', {}).items(): + package = NPMPackage( + registry=self.registry, name=name, version=version + ) + package.install() + + @cached_property + def metadata(self): + response = requests.get(url=self.url) + return response.json() + + @property + def url(self): + f = furl(self.registry.url) + f.path.segments = f.path.segments + [self.name] + return f.tostr() + + @property + def version_metadata(self): + return self.metadata['versions'][self.get_best_version()] + + @property + def versions(self): + return [ + force_bytes(version) for version in self.metadata['versions'].keys() + ] + + +class NPMRegistry(object): + def __init__(self, url=None, cache_path=None, module_directory=None, package_filename=None, lock_filename=None): + self.url = url or self.DEFAULT_REGISTRY_URL + self.cache_path = cache_path or mkdtemp() + self.module_directory = module_directory or self.DEFAULT_MODULE_DIRECTORY + self.package_file = package_filename or self.DEFAULT_PACKAGE_FILENAME + self.lock_filename = lock_filename or self.DEFAULT_LOCK_FILENAME + + def _install_package(self, name, version): + package = NPMPackage(registry=self, name=name, version=version) + package.install() + + def _read_package(self): + with self.package_file.open(mode='rb') as file_object: + self._package_data = json.loads(file_object.read()) + + def install(self, package=None): + if package: + name, version = package.split('@') + self._install_package(name=name, version=version) + else: + self._read_package() + + for name, version in self._package_data['dependencies'].items(): + self._install_package(name=name, version=version) + + +class JSDependencyManager(object): + def install(self, app_name=None): + if app_name: + app_config_list = [apps.get_app_config(app_label=app_name)] + else: + app_config_list = apps.get_app_configs() + + for app in app_config_list: + path = Path(app.path, 'static') + entries = list(path.glob('*/package.json')) + if entries: + print('Installing JavaScript packages for app: {}'.format(app.label)) + + for entry in entries: + npm_client = NPMRegistry( + module_directory=entry.parent / 'node_modules', + package_filename=entry + ) + npm_client.install() diff --git a/mayan/apps/dependencies/management/__init__.py b/mayan/apps/dependencies/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mayan/apps/dependencies/management/commands/__init__.py b/mayan/apps/dependencies/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mayan/apps/common/management/commands/installjavascript.py b/mayan/apps/dependencies/management/commands/installjavascript.py similarity index 100% rename from mayan/apps/common/management/commands/installjavascript.py rename to mayan/apps/dependencies/management/commands/installjavascript.py diff --git a/mayan/settings/base.py b/mayan/settings/base.py index e261dbb61d..9192d17cd2 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -89,6 +89,7 @@ INSTALLED_APPS = ( 'mayan.apps.common', 'mayan.apps.converter', 'mayan.apps.dashboards', + 'mayan.apps.dependencies', 'mayan.apps.django_gpg', 'mayan.apps.dynamic_search', 'mayan.apps.events',