diff --git a/.gitignore b/.gitignore index 3405a7a43e..b45fabd5cd 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ static_collected/ /mayan/media/static/ /venv/ /whoosh_index/ +node_modules/ diff --git a/HISTORY.rst b/HISTORY.rst index b70f6e3789..9d2d120824 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -133,7 +133,9 @@ of specifying their default path. - Unify checkbox selection code for list items and table items. - Add smart checkbox manager. - +- Update Chart.js version. +- Improve line chart appearance. Fix mouse hover label issue. +- Add JavaScript dependency manager. 2.7.3 (2017-09-11) ================== diff --git a/docs/releases/3.0.rst b/docs/releases/3.0.rst index c9ca68f2c3..4951839f88 100644 --- a/docs/releases/3.0.rst +++ b/docs/releases/3.0.rst @@ -310,6 +310,22 @@ This will select the first, the last and all items in between. To deselect multi items the same procedure is used. This code was donated by the Paperattor project (www.paperattor.com). +Add JavaScript dependency manager +--------------------------------- +An internal utility to install and upgrade the JavaScript dependencies was added. +This depency manager allows for the easier maintenace of the JavaScript libraries +used through the project. + +Previously JavaScript libraries we downloaded and installed by manually. These +libraries were them checked into the Git repository. Finally to enable them +the correspoding imports were added to the base templates in the apppeance app. + +This new manager is the first step to start resolving these issues. The manager +allows apps to specify their own dependencies. These dependecies are then +downloaded when the project is installed or upgraded. As such they are not +part of the repository and lower the file size of the project. + + Other changes worth mentioning ------------------------------ - Add Makefile target to check the format of the README.rst file. @@ -408,6 +424,9 @@ Other changes worth mentioning - Sort permission namespaces and permissions in the role permission views. - Invert the columns in the ACL detail view. - Remove the data filters feature. +- Update Chart.js version. +- Improve line chart appearance. Fix issue with mouse over labels next other chart margin. + Removals -------- diff --git a/mayan/apps/common/exceptions.py b/mayan/apps/common/exceptions.py index 54a297ca1f..a007bad142 100644 --- a/mayan/apps/common/exceptions.py +++ b/mayan/apps/common/exceptions.py @@ -14,3 +14,11 @@ class NotLatestVersion(BaseCommonException): """ def __init__(self, upstream_version): self.upstream_version = upstream_version + + +class NPMException(BaseCommonException): + """Base exception for the NPM registry client""" + + +class NPMPackgeIntegrityError(NPMException): + """Hash mismatch exception""" diff --git a/mayan/apps/common/javascript.py b/mayan/apps/common/javascript.py new file mode 100644 index 0000000000..401fdf23ee --- /dev/null +++ b/mayan/apps/common/javascript.py @@ -0,0 +1,150 @@ +from __future__ import unicode_literals + +import base64 +import hashlib +import json +import os +import shutil +import sys +import tarfile + +from furl import furl +import requests + +from django.apps import apps + +from .exceptions import NPMException, NPMPackgeIntegrityError + + +class NPMPackage(object): + def __init__(self, registry, name, version): + self.registry = registry + self.name = name + self.version = version + + def _download(self): + with requests.get(self.metadata['dist']['tarball'], stream=True) as response: + with open(name=self.tar_file_path, mode='wb') as file_object: + file_object.write(response.content) + + try: + upstream_algorithm_name, upstream_integrity_value = self.metadata['dist']['integrity'].split('-', 1) + except KeyError: + upstream_algorithm_name = 'sha1' + upstream_integrity_value = self.metadata['dist']['shasum'] + + algorithms = { + 'sha1': lambda data: hashlib.sha1(data).hexdigest(), + 'sha256': lambda data: base64.b64encode(hashlib.sha256(data).digest()), + 'sha512': lambda data: base64.b64encode(hashlib.sha512(data).digest()), + } + + try: + algorithm = algorithms[upstream_algorithm_name] + except KeyError: + raise NPMException('Unknown hash algorithm: {}'.format(upstream_algorithm_name)) + + with open(name=self.tar_file_path, mode='rb') as file_object: + integrity_value = algorithm(file_object.read()) + + if integrity_value != upstream_integrity_value: + os.unlink(self.tar_file_path) + raise NPMPackgeIntegrityError( + 'Hash of downloaded package doesn\'t match online version.' + ) + + def _extract(self): + shutil.rmtree( + os.path.join( + self.registry.module_directory, self.name + ), ignore_errors=True + ) + + with tarfile.open(name=self.tar_file_path, mode='r') as file_object: + file_object.extractall(path=self.registry.module_directory) + + os.rename( + os.path.join(self.registry.module_directory, 'package'), + os.path.join(self.registry.module_directory, self.name) + ) + + def install(self): + print 'Installing package: {}@{}'.format(self.name, self.version) + + self._download() + self._extract() + + for name, version in self.metadata.get('dependencies', {}).items(): + package = NPMPackage(registry=self.registry, name=name, version=version[1:]) + package.install() + + @property + def tar_filename(self): + if not hasattr(self, '_tar_filename'): + self._tar_filename = furl(self.metadata['dist']['tarball']).path.segments[-1] + + return self._tar_filename + + @property + def tar_file_path(self): + if not hasattr(self, '_tar_file_path'): + self._tar_file_path = os.path.join(self.registry.cache_path, self.tar_filename) + + return self._tar_file_path + + @property + def metadata(self): + if not hasattr(self, '_metadata'): + self._metadata = requests.get(url=self.get_url()).json() + return self._metadata + + def get_url(self): + f = furl(self.registry.url) + f.path.segments = f.path.segments + [self.name, self.version] + return f.tostr() + + +class NPMRegistry(object): + DEFAULT_CACHE_PATH = '/tmp' + DEFAULT_REGISTRY_URL = 'http://registry.npmjs.com' + DEFAULT_MODULE_DIRECTORY = 'node_modules' + DEFAULT_PACKAGE_FILENAME = 'package.json' + DEFAULT_LOCK_FILENAME = 'package-lock.json' + + def __init__(self, url=None, cache_path=None, module_directory=None, package_filename=None, lock_filename=None): + self.url = url or self.DEFAULT_REGISTRY_URL + self.cache_path = cache_path or self.DEFAULT_CACHE_PATH + self.module_directory = module_directory or self.DEFAULT_MODULE_DIRECTORY + self.package_file = package_filename or self.DEFAULT_PACKAGE_FILENAME + self.lock_filename = lock_filename or self.DEFAULT_LOCK_FILENAME + + def _install_package(self, name, version): + package = NPMPackage(registry=self, name=name, version=version) + package.install() + + def _read_package(self): + with open(self.package_file) as file_object: + self._package_data = json.loads(file_object.read()) + + def install(self, package=None): + if package: + name, version = package.split('@') + self._install_package(name=name, version=version) + else: + self._read_package() + + for name, version in self._package_data['dependencies'].items(): + self._install_package(name=name, version=version[1:]) + + +class JSDependencyManager(object): + def install(self): + for app in apps.get_app_configs(): + for root, dirs, files in os.walk(os.path.join(app.path, 'static')): + if 'package.json' in files and not any(map(lambda x: x in root, ['node_modules', 'packages', 'vendors'])): + print 'Installing JavaScript packages for app: {} - {}'.format(app.label, root) + npm_client = NPMRegistry( + module_directory=os.path.join(root, 'node_modules'), + package_filename=os.path.join(root, 'package.json') + ) + npm_client.install() diff --git a/mayan/apps/common/management/commands/initialsetup.py b/mayan/apps/common/management/commands/initialsetup.py index 2a91f83991..eadbe6c7cb 100644 --- a/mayan/apps/common/management/commands/initialsetup.py +++ b/mayan/apps/common/management/commands/initialsetup.py @@ -11,5 +11,6 @@ class Command(management.BaseCommand): def handle(self, *args, **options): management.call_command('createsettings', interactive=False) pre_initial_setup.send(sender=self) + management.call_command('installjavascript', interactive=False) management.call_command('createautoadmin', interactive=False) post_initial_setup.send(sender=self) diff --git a/mayan/apps/common/management/commands/installjavascript.py b/mayan/apps/common/management/commands/installjavascript.py new file mode 100644 index 0000000000..722b7fb6c2 --- /dev/null +++ b/mayan/apps/common/management/commands/installjavascript.py @@ -0,0 +1,13 @@ +from __future__ import unicode_literals + +from django.core import management + +from ...javascript import JSDependencyManager + + +class Command(management.BaseCommand): + help = 'Install JavaScript dependencies.' + + def handle(self, *args, **options): + js_manager = JSDependencyManager() + js_manager.install() diff --git a/mayan/apps/common/management/commands/performupgrade.py b/mayan/apps/common/management/commands/performupgrade.py index 9e78fa6e8b..497dc46a0f 100644 --- a/mayan/apps/common/management/commands/performupgrade.py +++ b/mayan/apps/common/management/commands/performupgrade.py @@ -17,6 +17,8 @@ class Command(management.BaseCommand): 'Error during pre_upgrade signal: %s' % exception ) + management.call_command('installjavascript', interactive=False) + try: perform_upgrade.send(sender=self) except Exception as exception: