diff --git a/mayan/apps/common/classes.py b/mayan/apps/common/classes.py index 2c52f2e16a..85de871f00 100644 --- a/mayan/apps/common/classes.py +++ b/mayan/apps/common/classes.py @@ -281,7 +281,6 @@ class Template(object): result.append(template.render(request=request)) return result - @classmethod def get(cls, name): return cls._registry[name] diff --git a/mayan/apps/common/exceptions.py b/mayan/apps/common/exceptions.py index e74b95ecf5..2576dc7435 100644 --- a/mayan/apps/common/exceptions.py +++ b/mayan/apps/common/exceptions.py @@ -42,11 +42,3 @@ class UnknownLatestVersion(BaseCommonException): """ It is not possible to determine what is the latest upstream version. """ - - -class NPMException(BaseCommonException): - """Base exception for the NPM registry client""" - - -class NPMPackgeIntegrityError(NPMException): - """Hash mismatch exception""" diff --git a/mayan/apps/common/javascript.py b/mayan/apps/common/javascript.py deleted file mode 100644 index eb9305835b..0000000000 --- a/mayan/apps/common/javascript.py +++ /dev/null @@ -1,194 +0,0 @@ -from __future__ import print_function, unicode_literals - -import base64 -import hashlib -import json -import os -import shutil -import tarfile - -from furl import furl -from pathlib2 import Path -import requests -from semver import max_satisfying - -from django.apps import apps -from django.utils.encoding import force_bytes - -from .exceptions import NPMException, NPMPackgeIntegrityError -from .utils import mkdtemp - - -class NPMPackage(object): - def __init__(self, registry, name, version): - self.registry = registry - self.name = name - self.version = version - - def _download(self): - with requests.get(self.download_metadata['dist']['tarball'], stream=True) as response: - with open(name=self.tar_file_path, mode='wb') as file_object: - file_object.write(response.content) - - try: - upstream_algorithm_name, upstream_integrity_value = self.download_metadata['dist']['integrity'].split('-', 1) - except KeyError: - upstream_algorithm_name = 'sha1' - upstream_integrity_value = self.download_metadata['dist']['shasum'] - - algorithms = { - 'sha1': lambda data: hashlib.sha1(data).hexdigest(), - 'sha256': lambda data: base64.b64encode(hashlib.sha256(data).digest()), - 'sha512': lambda data: base64.b64encode(hashlib.sha512(data).digest()), - } - - try: - algorithm = algorithms[upstream_algorithm_name] - except KeyError: - raise NPMException( - 'Unknown hash algorithm: {}'.format(upstream_algorithm_name) - ) - - with open(name=self.tar_file_path, mode='rb') as file_object: - integrity_value = algorithm(file_object.read()) - - if integrity_value != upstream_integrity_value: - os.unlink(self.tar_file_path) - raise NPMPackgeIntegrityError( - 'Hash of downloaded package doesn\'t match online version.' - ) - - def _extract(self): - shutil.rmtree( - os.path.join( - self.registry.module_directory, self.name - ), ignore_errors=True - ) - - with tarfile.open(name=self.tar_file_path, mode='r') as file_object: - file_object.extractall(path=self.registry.module_directory) - - os.rename( - os.path.join(self.registry.module_directory, 'package'), - os.path.join(self.registry.module_directory, self.name) - ) - - @property - def best_version(self): - if not hasattr(self, '_best_version'): - self._best_version = max_satisfying( - self.versions, force_bytes(self.version), loose=True - ) - print('Best version: {}'.format(self._best_version)) - - return self._best_version - - @property - def download_metadata(self): - if not hasattr(self, '_download_metadata'): - response = requests.get(url=self.download_url) - self._download_metadata = response.json() - return self._download_metadata - - @property - def download_url(self): - f = furl(self.url) - f.path.segments = f.path.segments + [self.best_version] - return f.tostr() - - def install(self, include_dependencies=False): - print('Installing package: {}{}'.format(self.name, self.version)) - - self._download() - self._extract() - - if include_dependencies: - for name, version in self.download_metadata.get('dependencies', {}).items(): - package = NPMPackage( - registry=self.registry, name=name, version=version - ) - package.install() - - @property - def tar_filename(self): - if not hasattr(self, '_tar_filename'): - self._tar_filename = furl( - self.download_metadata['dist']['tarball'] - ).path.segments[-1] - - return self._tar_filename - - @property - def tar_file_path(self): - if not hasattr(self, '_tar_file_path'): - self._tar_file_path = os.path.join( - self.registry.cache_path, self.tar_filename - ) - - return self._tar_file_path - - @property - def url(self): - f = furl(self.registry.url) - f.path.segments = f.path.segments + [self.name] - return f.tostr() - - @property - def versions(self): - if not hasattr(self, '_versions'): - response = requests.get(url=self.url) - self._versions = [ - force_bytes(version) for version in response.json()['versions'].keys() - ] - - return self._versions - - -class NPMRegistry(object): - DEFAULT_REGISTRY_URL = 'http://registry.npmjs.com' - DEFAULT_MODULE_DIRECTORY = 'node_modules' - DEFAULT_PACKAGE_FILENAME = 'package.json' - DEFAULT_LOCK_FILENAME = 'package-lock.json' - - def __init__(self, url=None, cache_path=None, module_directory=None, package_filename=None, lock_filename=None): - self.url = url or self.DEFAULT_REGISTRY_URL - self.cache_path = cache_path or mkdtemp() - self.module_directory = module_directory or self.DEFAULT_MODULE_DIRECTORY - self.package_file = package_filename or self.DEFAULT_PACKAGE_FILENAME - self.lock_filename = lock_filename or self.DEFAULT_LOCK_FILENAME - - def _install_package(self, name, version): - package = NPMPackage(registry=self, name=name, version=version) - package.install() - - def _read_package(self): - with open(self.package_file) as file_object: - self._package_data = json.loads(file_object.read()) - - def install(self, package=None): - if package: - name, version = package.split('@') - self._install_package(name=name, version=version) - else: - self._read_package() - - for name, version in self._package_data['dependencies'].items(): - self._install_package(name=name, version=version) - - -class JSDependencyManager(object): - def install(self, app_name=None): - if app_name: - app_config_list = [apps.get_app_config(app_label=app_name)] - else: - app_config_list = apps.get_app_configs() - - for app in app_config_list: - for root, dirs, files in os.walk(os.path.join(app.path, 'static')): - if 'package.json' in files and not (set(Path(root).parts) & set(['node_modules', 'packages', 'vendors'])): - print('Installing JavaScript packages for app: {} - {}'.format(app.label, root)) - npm_client = NPMRegistry( - module_directory=os.path.join(root, 'node_modules'), - package_filename=os.path.join(root, 'package.json') - ) - npm_client.install() diff --git a/mayan/apps/common/management/commands/initialsetup.py b/mayan/apps/common/management/commands/initialsetup.py index 456983279c..24647167e7 100644 --- a/mayan/apps/common/management/commands/initialsetup.py +++ b/mayan/apps/common/management/commands/initialsetup.py @@ -27,11 +27,6 @@ class Command(management.BaseCommand): help='Force execution of the initialization process.', ) - parser.add_argument( - '--no-javascript', action='store_true', dest='no_javascript', - help='Don\'t download the JavaScript dependencies.', - ) - def initialize_system(self, force=False): system_path = os.path.join(settings.MEDIA_ROOT, SYSTEM_DIR) settings_path = os.path.join(settings.MEDIA_ROOT, 'mayan_settings') @@ -88,8 +83,5 @@ class Command(management.BaseCommand): self.initialize_system(force=options.get('force', False)) pre_initial_setup.send(sender=self) - if not options.get('no_javascript', False): - management.call_command('installjavascript', interactive=False) - management.call_command('createautoadmin', interactive=False) post_initial_setup.send(sender=self) diff --git a/mayan/apps/common/management/commands/performupgrade.py b/mayan/apps/common/management/commands/performupgrade.py index d2ba4999c8..9e94e7360b 100644 --- a/mayan/apps/common/management/commands/performupgrade.py +++ b/mayan/apps/common/management/commands/performupgrade.py @@ -19,8 +19,6 @@ class Command(management.BaseCommand): ) ) - management.call_command('installjavascript', interactive=False) - try: perform_upgrade.send(sender=self) except Exception as exception: diff --git a/mayan/apps/dependencies/__init__.py b/mayan/apps/dependencies/__init__.py new file mode 100644 index 0000000000..790c8ccc9f --- /dev/null +++ b/mayan/apps/dependencies/__init__.py @@ -0,0 +1,3 @@ +from __future__ import unicode_literals + +default_app_config = 'mayan.apps.dependencies.apps.DependenciesApp' diff --git a/mayan/apps/dependencies/apps.py b/mayan/apps/dependencies/apps.py new file mode 100644 index 0000000000..0f6850a5a5 --- /dev/null +++ b/mayan/apps/dependencies/apps.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common import MayanAppConfig +from mayan.apps.common.signals import post_initial_setup, post_upgrade + +from .handlers import handler_install_javascript + + +class DependenciesApp(MayanAppConfig): + app_namespace = 'dependencies' + app_url = 'dependencies' + has_rest_api = False + has_tests = False + name = 'mayan.apps.dependencies' + verbose_name = _('Dependencies') + + def ready(self): + super(DependenciesApp, self).ready() + + post_initial_setup.connect( + dispatch_uid='dependendies_handler_install_javascript', + receiver=handler_install_javascript + ) + post_upgrade.connect( + dispatch_uid='dependendies_handler_install_javascript', + receiver=handler_install_javascript + ) diff --git a/mayan/apps/dependencies/exceptions.py b/mayan/apps/dependencies/exceptions.py new file mode 100644 index 0000000000..4a7799a31e --- /dev/null +++ b/mayan/apps/dependencies/exceptions.py @@ -0,0 +1,7 @@ +from __future__ import unicode_literals + + +class DependenciesException(Exception): + """ + Base exception for the dependencies app + """ diff --git a/mayan/apps/dependencies/handlers.py b/mayan/apps/dependencies/handlers.py new file mode 100644 index 0000000000..00ccc8ff42 --- /dev/null +++ b/mayan/apps/dependencies/handlers.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +from .javascript import JSDependencyManager + + +def handler_install_javascript(sender, **kwargs): + js_manager = JSDependencyManager() + js_manager.install() diff --git a/mayan/apps/dependencies/javascript.py b/mayan/apps/dependencies/javascript.py new file mode 100644 index 0000000000..6f6806d7b8 --- /dev/null +++ b/mayan/apps/dependencies/javascript.py @@ -0,0 +1,188 @@ +from __future__ import print_function, unicode_literals + +import base64 +import hashlib +import json +import shutil +import tarfile + +from furl import furl +from pathlib2 import Path +import requests +from semver import max_satisfying + +from django.apps import apps +from django.utils.encoding import force_bytes, force_text +from django.utils.functional import cached_property + +from mayan.apps.common.utils import mkdtemp + +from .exceptions import DependenciesException + +DEFAULT_REGISTRY_URL = 'http://registry.npmjs.com' +DEFAULT_MODULE_DIRECTORY = 'node_modules' +DEFAULT_PACKAGE_FILENAME = 'package.json' +DEFAULT_LOCK_FILENAME = 'package-lock.json' + + +class NPMPackage(object): + def __init__(self, registry, name, version): + self.registry = registry + self.name = name + self.version = version + + def download(self): + algorithm_function = self.get_algorithm_function() + tar_file_path = self.get_tar_file_path() + + with requests.get(self.version_metadata['dist']['tarball'], stream=True) as response: + with tar_file_path.open(mode='wb') as file_object: + shutil.copyfileobj(response.raw, file_object) + + with tar_file_path.open(mode='rb') as file_object: + integrity_is_good = algorithm_function(file_object.read()) + + if not integrity_is_good: + tar_file_path.unlink() + raise DependenciesException( + 'Hash of downloaded package doesn\'t match online version.' + ) + + def extract(self): + shutil.rmtree( + path=force_text( + Path( + self.registry.module_directory, self.name + ) + ), ignore_errors=True + ) + + with tarfile.open(name=force_text(self.get_tar_file_path()), mode='r') as file_object: + file_object.extractall( + path=force_text(self.registry.module_directory) + ) + + Path(self.registry.module_directory, 'package').rename( + target=Path(self.registry.module_directory, self.name) + ) + + def get_algorithm_function(self): + try: + integrity = self.version_metadata['dist']['integrity'] + except KeyError: + algorithm_name = 'sha1' + integrity_value = self.version_metadata['dist']['shasum'] + else: + algorithm_name, integrity_value = integrity.split('-', 1) + + algorithms = { + 'sha1': lambda data: hashlib.sha1(data).hexdigest() == integrity_value, + 'sha256': lambda data: base64.b64encode(hashlib.sha256(data).digest()) == integrity_value, + 'sha512': lambda data: base64.b64encode(hashlib.sha512(data).digest()) == integrity_value, + } + + try: + algorithm = algorithms[algorithm_name] + except KeyError: + raise DependenciesException( + 'Unknown hash algorithm: {}'.format(algorithm_name) + ) + else: + return algorithm + + def get_best_version(self): + return max_satisfying( + self.versions, force_bytes(self.version), loose=True + ) + + def get_tar_file_path(self): + return Path( + self.registry.cache_path, self.get_tar_filename() + ) + + def get_tar_filename(self): + return furl( + self.version_metadata['dist']['tarball'] + ).path.segments[-1] + + def install(self, include_dependencies=False): + print('Installing package: {}{}'.format(self.name, self.version)) + + self.download() + self.extract() + + if include_dependencies: + for name, version in self.version_metadata.get('dependencies', {}).items(): + package = NPMPackage( + registry=self.registry, name=name, version=version + ) + package.install() + + @cached_property + def metadata(self): + response = requests.get(url=self.url) + return response.json() + + @property + def url(self): + f = furl(self.registry.url) + f.path.segments = f.path.segments + [self.name] + return f.tostr() + + @property + def version_metadata(self): + return self.metadata['versions'][self.get_best_version()] + + @property + def versions(self): + return [ + force_bytes(version) for version in self.metadata['versions'].keys() + ] + + +class NPMRegistry(object): + def __init__(self, url=None, cache_path=None, module_directory=None, package_filename=None, lock_filename=None): + self.url = url or self.DEFAULT_REGISTRY_URL + self.cache_path = cache_path or mkdtemp() + self.module_directory = module_directory or self.DEFAULT_MODULE_DIRECTORY + self.package_file = package_filename or self.DEFAULT_PACKAGE_FILENAME + self.lock_filename = lock_filename or self.DEFAULT_LOCK_FILENAME + + def _install_package(self, name, version): + package = NPMPackage(registry=self, name=name, version=version) + package.install() + + def _read_package(self): + with self.package_file.open(mode='rb') as file_object: + self._package_data = json.loads(file_object.read()) + + def install(self, package=None): + if package: + name, version = package.split('@') + self._install_package(name=name, version=version) + else: + self._read_package() + + for name, version in self._package_data['dependencies'].items(): + self._install_package(name=name, version=version) + + +class JSDependencyManager(object): + def install(self, app_name=None): + if app_name: + app_config_list = [apps.get_app_config(app_label=app_name)] + else: + app_config_list = apps.get_app_configs() + + for app in app_config_list: + path = Path(app.path, 'static') + entries = list(path.glob('*/package.json')) + if entries: + print('Installing JavaScript packages for app: {}'.format(app.label)) + + for entry in entries: + npm_client = NPMRegistry( + module_directory=entry.parent / 'node_modules', + package_filename=entry + ) + npm_client.install() diff --git a/mayan/apps/dependencies/management/__init__.py b/mayan/apps/dependencies/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mayan/apps/dependencies/management/commands/__init__.py b/mayan/apps/dependencies/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mayan/apps/common/management/commands/installjavascript.py b/mayan/apps/dependencies/management/commands/installjavascript.py similarity index 100% rename from mayan/apps/common/management/commands/installjavascript.py rename to mayan/apps/dependencies/management/commands/installjavascript.py diff --git a/mayan/settings/base.py b/mayan/settings/base.py index e261dbb61d..9192d17cd2 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -89,6 +89,7 @@ INSTALLED_APPS = ( 'mayan.apps.common', 'mayan.apps.converter', 'mayan.apps.dashboards', + 'mayan.apps.dependencies', 'mayan.apps.django_gpg', 'mayan.apps.dynamic_search', 'mayan.apps.events',