matthias/mayan-edms
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update and move JavaScript install code

Browse Source 
Update the JavaScript dependency installation code to handle scoped
packages. The code is also updated to use pathlib's Path.

Move the JavaScript dependency installation to its own app named
dependencies.

Signed-off-by: Roberto Rosario <Roberto.Rosario@mayan-edms.com>
This commit is contained in:
Roberto Rosario
2018-12-26 02:15:44 -04:00
parent d6c7a0d765
commit 4ff9794286
14 changed files with 236 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
mayan/apps/common/classes.py
View File

@@ -281,7 +281,6 @@ class Template(object):
result.append(template.render(request=request))
return result
@classmethod
def get(cls, name):
return cls._registry[name]

8
mayan/apps/common/exceptions.py
View File

@@ -42,11 +42,3 @@ class UnknownLatestVersion(BaseCommonException):
"""
It is not possible to determine what is the latest upstream version.
"""
class NPMException(BaseCommonException):
"""Base exception for the NPM registry client"""
class NPMPackgeIntegrityError(NPMException):
"""Hash mismatch exception"""

194
mayan/apps/common/javascript.py
View File

@@ -1,194 +0,0 @@
from __future__ import print_function, unicode_literals
import base64
import hashlib
import json
import os
import shutil
import tarfile
from furl import furl
from pathlib2 import Path
import requests
from semver import max_satisfying
from django.apps import apps
from django.utils.encoding import force_bytes
from .exceptions import NPMException, NPMPackgeIntegrityError
from .utils import mkdtemp
class NPMPackage(object):
def __init__(self, registry, name, version):
self.registry = registry
self.name = name
self.version = version
def _download(self):
with requests.get(self.download_metadata['dist']['tarball'], stream=True) as response:
with open(name=self.tar_file_path, mode='wb') as file_object:
file_object.write(response.content)
try:
upstream_algorithm_name, upstream_integrity_value = self.download_metadata['dist']['integrity'].split('-', 1)
except KeyError:
upstream_algorithm_name = 'sha1'
upstream_integrity_value = self.download_metadata['dist']['shasum']
algorithms = {
'sha1': lambda data: hashlib.sha1(data).hexdigest(),
'sha256': lambda data: base64.b64encode(hashlib.sha256(data).digest()),
'sha512': lambda data: base64.b64encode(hashlib.sha512(data).digest()),
}
try:
algorithm = algorithms[upstream_algorithm_name]
except KeyError:
raise NPMException(
'Unknown hash algorithm: {}'.format(upstream_algorithm_name)
)
with open(name=self.tar_file_path, mode='rb') as file_object:
integrity_value = algorithm(file_object.read())
if integrity_value != upstream_integrity_value:
os.unlink(self.tar_file_path)
raise NPMPackgeIntegrityError(
'Hash of downloaded package doesn\'t match online version.'
)
def _extract(self):
shutil.rmtree(
os.path.join(
self.registry.module_directory, self.name
), ignore_errors=True
)
with tarfile.open(name=self.tar_file_path, mode='r') as file_object:
file_object.extractall(path=self.registry.module_directory)
os.rename(
os.path.join(self.registry.module_directory, 'package'),
os.path.join(self.registry.module_directory, self.name)
)
@property
def best_version(self):
if not hasattr(self, '_best_version'):
self._best_version = max_satisfying(
self.versions, force_bytes(self.version), loose=True
)
print('Best version: {}'.format(self._best_version))
return self._best_version
@property
def download_metadata(self):
if not hasattr(self, '_download_metadata'):
response = requests.get(url=self.download_url)
self._download_metadata = response.json()
return self._download_metadata
@property
def download_url(self):
f = furl(self.url)
f.path.segments = f.path.segments + [self.best_version]
return f.tostr()
def install(self, include_dependencies=False):
print('Installing package: {}{}'.format(self.name, self.version))
self._download()
self._extract()
if include_dependencies:
for name, version in self.download_metadata.get('dependencies', {}).items():
package = NPMPackage(
registry=self.registry, name=name, version=version
)
package.install()
@property
def tar_filename(self):
if not hasattr(self, '_tar_filename'):
self._tar_filename = furl(
self.download_metadata['dist']['tarball']
).path.segments[-1]
return self._tar_filename
@property
def tar_file_path(self):
if not hasattr(self, '_tar_file_path'):
self._tar_file_path = os.path.join(
self.registry.cache_path, self.tar_filename
)
return self._tar_file_path
@property
def url(self):
f = furl(self.registry.url)
f.path.segments = f.path.segments + [self.name]
return f.tostr()
@property
def versions(self):
if not hasattr(self, '_versions'):
response = requests.get(url=self.url)
self._versions = [
force_bytes(version) for version in response.json()['versions'].keys()
]
return self._versions
class NPMRegistry(object):
DEFAULT_REGISTRY_URL = 'http://registry.npmjs.com'
DEFAULT_MODULE_DIRECTORY = 'node_modules'
DEFAULT_PACKAGE_FILENAME = 'package.json'
DEFAULT_LOCK_FILENAME = 'package-lock.json'
def __init__(self, url=None, cache_path=None, module_directory=None, package_filename=None, lock_filename=None):
self.url = url or self.DEFAULT_REGISTRY_URL
self.cache_path = cache_path or mkdtemp()
self.module_directory = module_directory or self.DEFAULT_MODULE_DIRECTORY
self.package_file = package_filename or self.DEFAULT_PACKAGE_FILENAME
self.lock_filename = lock_filename or self.DEFAULT_LOCK_FILENAME
def _install_package(self, name, version):
package = NPMPackage(registry=self, name=name, version=version)
package.install()
def _read_package(self):
with open(self.package_file) as file_object:
self._package_data = json.loads(file_object.read())
def install(self, package=None):
if package:
name, version = package.split('@')
self._install_package(name=name, version=version)
else:
self._read_package()
for name, version in self._package_data['dependencies'].items():
self._install_package(name=name, version=version)
class JSDependencyManager(object):
def install(self, app_name=None):
if app_name:
app_config_list = [apps.get_app_config(app_label=app_name)]
else:
app_config_list = apps.get_app_configs()
for app in app_config_list:
for root, dirs, files in os.walk(os.path.join(app.path, 'static')):
if 'package.json' in files and not (set(Path(root).parts) & set(['node_modules', 'packages', 'vendors'])):
print('Installing JavaScript packages for app: {} - {}'.format(app.label, root))
npm_client = NPMRegistry(
module_directory=os.path.join(root, 'node_modules'),
package_filename=os.path.join(root, 'package.json')
)
npm_client.install()

8
mayan/apps/common/management/commands/initialsetup.py
View File

@@ -27,11 +27,6 @@ class Command(management.BaseCommand):
help='Force execution of the initialization process.',
)
parser.add_argument(
'--no-javascript', action='store_true', dest='no_javascript',
help='Don\'t download the JavaScript dependencies.',
)
def initialize_system(self, force=False):
system_path = os.path.join(settings.MEDIA_ROOT, SYSTEM_DIR)
settings_path = os.path.join(settings.MEDIA_ROOT, 'mayan_settings')
@@ -88,8 +83,5 @@ class Command(management.BaseCommand):
self.initialize_system(force=options.get('force', False))
pre_initial_setup.send(sender=self)
if not options.get('no_javascript', False):
management.call_command('installjavascript', interactive=False)
management.call_command('createautoadmin', interactive=False)
post_initial_setup.send(sender=self)

2
mayan/apps/common/management/commands/performupgrade.py
View File

@@ -19,8 +19,6 @@ class Command(management.BaseCommand):
)
)
management.call_command('installjavascript', interactive=False)
try:
perform_upgrade.send(sender=self)
except Exception as exception:

3
mayan/apps/dependencies/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from __future__ import unicode_literals
default_app_config = 'mayan.apps.dependencies.apps.DependenciesApp'

29
mayan/apps/dependencies/apps.py Normal file
View File

@@ -0,0 +1,29 @@
from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mayan.apps.common import MayanAppConfig
from mayan.apps.common.signals import post_initial_setup, post_upgrade
from .handlers import handler_install_javascript
class DependenciesApp(MayanAppConfig):
app_namespace = 'dependencies'
app_url = 'dependencies'
has_rest_api = False
has_tests = False
name = 'mayan.apps.dependencies'
verbose_name = _('Dependencies')
def ready(self):
super(DependenciesApp, self).ready()
post_initial_setup.connect(
dispatch_uid='dependendies_handler_install_javascript',
receiver=handler_install_javascript
)
post_upgrade.connect(
dispatch_uid='dependendies_handler_install_javascript',
receiver=handler_install_javascript
)

7
mayan/apps/dependencies/exceptions.py Normal file
View File

@@ -0,0 +1,7 @@
from __future__ import unicode_literals
class DependenciesException(Exception):
"""
Base exception for the dependencies app
"""

8
mayan/apps/dependencies/handlers.py Normal file
View File

@@ -0,0 +1,8 @@
from __future__ import unicode_literals
from .javascript import JSDependencyManager
def handler_install_javascript(sender, **kwargs):
js_manager = JSDependencyManager()
js_manager.install()

188
mayan/apps/dependencies/javascript.py Normal file
View File

@@ -0,0 +1,188 @@
from __future__ import print_function, unicode_literals
import base64
import hashlib
import json
import shutil
import tarfile
from furl import furl
from pathlib2 import Path
import requests
from semver import max_satisfying
from django.apps import apps
from django.utils.encoding import force_bytes, force_text
from django.utils.functional import cached_property
from mayan.apps.common.utils import mkdtemp
from .exceptions import DependenciesException
DEFAULT_REGISTRY_URL = 'http://registry.npmjs.com'
DEFAULT_MODULE_DIRECTORY = 'node_modules'
DEFAULT_PACKAGE_FILENAME = 'package.json'
DEFAULT_LOCK_FILENAME = 'package-lock.json'
class NPMPackage(object):
def __init__(self, registry, name, version):
self.registry = registry
self.name = name
self.version = version
def download(self):
algorithm_function = self.get_algorithm_function()
tar_file_path = self.get_tar_file_path()
with requests.get(self.version_metadata['dist']['tarball'], stream=True) as response:
with tar_file_path.open(mode='wb') as file_object:
shutil.copyfileobj(response.raw, file_object)
with tar_file_path.open(mode='rb') as file_object:
integrity_is_good = algorithm_function(file_object.read())
if not integrity_is_good:
tar_file_path.unlink()
raise DependenciesException(
'Hash of downloaded package doesn\'t match online version.'
)
def extract(self):
shutil.rmtree(
path=force_text(
Path(
self.registry.module_directory, self.name
)
), ignore_errors=True
)
with tarfile.open(name=force_text(self.get_tar_file_path()), mode='r') as file_object:
file_object.extractall(
path=force_text(self.registry.module_directory)
)
Path(self.registry.module_directory, 'package').rename(
target=Path(self.registry.module_directory, self.name)
)
def get_algorithm_function(self):
try:
integrity = self.version_metadata['dist']['integrity']
except KeyError:
algorithm_name = 'sha1'
integrity_value = self.version_metadata['dist']['shasum']
else:
algorithm_name, integrity_value = integrity.split('-', 1)
algorithms = {
'sha1': lambda data: hashlib.sha1(data).hexdigest() == integrity_value,
'sha256': lambda data: base64.b64encode(hashlib.sha256(data).digest()) == integrity_value,
'sha512': lambda data: base64.b64encode(hashlib.sha512(data).digest()) == integrity_value,
}
try:
algorithm = algorithms[algorithm_name]
except KeyError:
raise DependenciesException(
'Unknown hash algorithm: {}'.format(algorithm_name)
)
else:
return algorithm
def get_best_version(self):
return max_satisfying(
self.versions, force_bytes(self.version), loose=True
)
def get_tar_file_path(self):
return Path(
self.registry.cache_path, self.get_tar_filename()
)
def get_tar_filename(self):
return furl(
self.version_metadata['dist']['tarball']
).path.segments[-1]
def install(self, include_dependencies=False):
print('Installing package: {}{}'.format(self.name, self.version))
self.download()
self.extract()
if include_dependencies:
for name, version in self.version_metadata.get('dependencies', {}).items():
package = NPMPackage(
registry=self.registry, name=name, version=version
)
package.install()
@cached_property
def metadata(self):
response = requests.get(url=self.url)
return response.json()
@property
def url(self):
f = furl(self.registry.url)
f.path.segments = f.path.segments + [self.name]
return f.tostr()
@property
def version_metadata(self):
return self.metadata['versions'][self.get_best_version()]
@property
def versions(self):
return [
force_bytes(version) for version in self.metadata['versions'].keys()
]
class NPMRegistry(object):
def __init__(self, url=None, cache_path=None, module_directory=None, package_filename=None, lock_filename=None):
self.url = url or self.DEFAULT_REGISTRY_URL
self.cache_path = cache_path or mkdtemp()
self.module_directory = module_directory or self.DEFAULT_MODULE_DIRECTORY
self.package_file = package_filename or self.DEFAULT_PACKAGE_FILENAME
self.lock_filename = lock_filename or self.DEFAULT_LOCK_FILENAME
def _install_package(self, name, version):
package = NPMPackage(registry=self, name=name, version=version)
package.install()
def _read_package(self):
with self.package_file.open(mode='rb') as file_object:
self._package_data = json.loads(file_object.read())
def install(self, package=None):
if package:
name, version = package.split('@')
self._install_package(name=name, version=version)
else:
self._read_package()
for name, version in self._package_data['dependencies'].items():
self._install_package(name=name, version=version)
class JSDependencyManager(object):
def install(self, app_name=None):
if app_name:
app_config_list = [apps.get_app_config(app_label=app_name)]
else:
app_config_list = apps.get_app_configs()
for app in app_config_list:
path = Path(app.path, 'static')
entries = list(path.glob('*/package.json'))
if entries:
print('Installing JavaScript packages for app: {}'.format(app.label))
for entry in entries:
npm_client = NPMRegistry(
module_directory=entry.parent / 'node_modules',
package_filename=entry
)
npm_client.install()

0
mayan/apps/dependencies/management/__init__.py Normal file
View File

0
mayan/apps/dependencies/management/commands/__init__.py Normal file
View File

0
mayan/apps/common/management/commands/installjavascript.py → mayan/apps/dependencies/management/commands/installjavascript.py
View File

1
mayan/settings/base.py
View File

@@ -89,6 +89,7 @@ INSTALLED_APPS = (
'mayan.apps.common',
'mayan.apps.converter',
'mayan.apps.dashboards',
'mayan.apps.dependencies',
'mayan.apps.django_gpg',
'mayan.apps.dynamic_search',
'mayan.apps.events',