From 75a6a211a833113ba0ae9ce61a5159b337149d3c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 25 Jan 2013 22:00:50 -0400 Subject: [PATCH 1/3] Split installation properties into namespaces --- apps/installation/classes.py | 75 +++++++++++++++++ apps/installation/models.py | 158 ++++++++++++++--------------------- 2 files changed, 139 insertions(+), 94 deletions(-) create mode 100644 apps/installation/classes.py diff --git a/apps/installation/classes.py b/apps/installation/classes.py new file mode 100644 index 0000000000..bbc0df076f --- /dev/null +++ b/apps/installation/classes.py @@ -0,0 +1,75 @@ +from django.utils.simplejson import dumps + + +class PropertyNamespace(object): + _registry = {} + + @classmethod + def get(cls, name): + return cls._registry[name] + + @classmethod + def get_all(cls): + return cls._registry.values() + + def __init__(self, name, label): + self.name = name + self.label = label + self.properties = {} + self.__class__._registry[name] = self + + def __unicode__(self): + return unicode(self.label) + + def __str__(self): + return str(self.label) + + def add_property(self, *args, **kwargs): + prop = Property(*args, **kwargs) + self.properties[prop.name] = prop + + def get_properties(self): + return self.properties.values() + + @property + def id(self): + return self.name + + +class Property(object): + _registry = {} + + @classmethod + def get_all(cls): + return cls._registry.values() + + @classmethod + def get(cls, name): + return cls._registry[name] + + @classmethod + def get_reportable(cls, as_dict=False, as_json=False): + if as_json: + return dumps(cls.get_reportable(as_dict=True)) + + if not as_dict: + return [prop for prop in cls.get_all() if prop.report] + else: + result = {} + for prop in cls.get_all(): + if prop.report: + result[prop.name] = unicode(prop.value) + return result + + def __init__(self, name, label, value, report=False): + self.name = name + self.label = label + self.value = value + self.report = report + self.__class__._registry[name] = self + + def __unicode__(self): + return unicode(self.value) + + def __str__(self): + return str(self.value) diff --git a/apps/installation/models.py b/apps/installation/models.py index fca070af37..1779b83c14 100644 --- a/apps/installation/models.py +++ b/apps/installation/models.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import os import sys import platform @@ -20,7 +22,6 @@ else: from django.db import models from django.utils.translation import ugettext_lazy as _ from django.utils.datastructures import SortedDict -from django.utils.simplejson import dumps from django.conf import settings from common.models import Singleton @@ -29,6 +30,8 @@ from main import __version__ as mayan_version from lock_manager import Lock, LockError from ocr.conf.settings import TESSERACT_PATH, UNPAPER_PATH, PDFTOTEXT_PATH +from .classes import Property, PropertyNamespace + FORM_SUBMIT_URL = 'https://docs.google.com/spreadsheet/formResponse' FORM_KEY = 'dGZrYkw3SDl5OENMTG15emp1UFFEUWc6MQ' FORM_RECEIVER_FIELD = 'entry.0.single' @@ -36,19 +39,6 @@ TIMEOUT = 5 FABFILE_MARKER = os.path.join(settings.PROJECT_ROOT, 'fabfile_install') -class Property(object): - def __init__(self, name, label, value): - self.name = name - self.label = label - self.value = value - - def __unicode__(self): - return unicode(self.value) - - def __str__(self): - return str(self.value) - - class Installation(Singleton): _properties = SortedDict() @@ -62,72 +52,89 @@ class Installation(Singleton): self.set_properties() return self._properties.values() - def set_properties(self): - self._properties = SortedDict() + def os_properties(self): + namespace = PropertyNamespace('os', _(u'Operating system')) if LSB: - self.add_property(Property('is_lsb', _(u'LSB OS'), True)) - self.add_property(Property('distributor_id', _(u'Distributor ID'), lsb_release('-i', '-s'))) - self.add_property(Property('description', _(u'Description'), lsb_release('-d', '-s'))) - self.add_property(Property('release', _(u'Release'), lsb_release('-r', '-s'))) - self.add_property(Property('codename', _(u'Codename'), lsb_release('-c', '-s'))) - self.add_property(Property('sysinfo', _(u'System info'), uname('-a'))) + namespace.add_property('is_lsb', _(u'LSB OS'), True, True) + namespace.add_property('distributor_id', _(u'Distributor ID'), lsb_release('-i', '-s'), True) + namespace.add_property('description', _(u'Description'), lsb_release('-d', '-s'), True) + namespace.add_property('release', _(u'Release'), lsb_release('-r', '-s'), True) + namespace.add_property('codename', _(u'Codename'), lsb_release('-c', '-s'), True) + namespace.add_property('sysinfo', _(u'System info'), uname('-a'), True) else: - self.add_property(Property('is_lsb', _(u'LSB OS'), False)) + namespace.add_property('is_lsb', _(u'LSB OS'), False) + + namespace.add_property('architecture', _(u'OS architecture'), platform.architecture(), report=True) + namespace.add_property('python_version', _(u'Python version'), platform.python_version(), report=True) + namespace.add_property('hostname', _(u'Hostname'), platform.node()) + namespace.add_property('platform', _(u'Platform'), sys.platform, report=True) + namespace.add_property('machine', _(u'Machine'), platform.machine(), report=True) + namespace.add_property('processor', _(u'Processor'), platform.processor(), report=True) + namespace.add_property('cpus', _(u'Number of CPUs'), psutil.NUM_CPUS, report=True) + namespace.add_property('total_phymem', _(u'Total physical memory'), pretty_size(psutil.TOTAL_PHYMEM), report=True) + namespace.add_property('disk_partitions', _(u'Disk partitions'), '; '.join(['%s %s %s %s' % (partition.device, partition.mountpoint, partition.fstype, partition.opts) for partition in psutil.disk_partitions()])) - self.add_property(Property('architecture', _(u'OS architecture'), platform.architecture())) - self.add_property(Property('python_version', _(u'Python version'), platform.python_version())) - self.add_property(Property('hostname', _(u'Hostname'), platform.node())) - self.add_property(Property('platform', _(u'Platform'), sys.platform)) - self.add_property(Property('machine', _(u'Machine'), platform.machine())) - self.add_property(Property('processor', _(u'Processor'), platform.processor())) - self.add_property(Property('cpus', _(u'Number of CPUs'), psutil.NUM_CPUS)) - self.add_property(Property('total_phymem', _(u'Total physical memory'), pretty_size(psutil.TOTAL_PHYMEM))) - self.add_property(Property('disk_partitions', _(u'Disk partitions'), '; '.join(['%s %s %s %s' % (partition.device, partition.mountpoint, partition.fstype, partition.opts) for partition in psutil.disk_partitions()]))) + def binary_dependencies(self): + namespace = PropertyNamespace('bins', _(u'Binary dependencies')) tesseract = pbs.Command(TESSERACT_PATH) try: - self.add_property(Property('tesseract', _(u'tesseract version'), tesseract('-v').stderr)) + namespace.add_property('tesseract', _(u'tesseract version'), tesseract('-v').stderr, report=True) except pbs.CommandNotFound: - self.add_property(Property('tesseract', _(u'tesseract version'), _(u'not found'))) + namespace.add_property('tesseract', _(u'tesseract version'), _(u'not found'), report=True) except Exception: - self.add_property(Property('tesseract', _(u'tesseract version'), _(u'error getting version'))) + namespace.add_property('tesseract', _(u'tesseract version'), _(u'error getting version'), report=True) unpaper = pbs.Command(UNPAPER_PATH) try: - self.add_property(Property('unpaper', _(u'unpaper version'), unpaper('-V').stdout)) + namespace.add_property('unpaper', _(u'unpaper version'), unpaper('-V').stdout, report=True) except pbs.CommandNotFound: - self.add_property(Property('unpaper', _(u'unpaper version'), _(u'not found'))) + namespace.add_property('unpaper', _(u'unpaper version'), _(u'not found'), report=True) except Exception: - self.add_property(Property('unpaper', _(u'unpaper version'), _(u'error getting version'))) + namespace.add_property('unpaper', _(u'unpaper version'), _(u'error getting version'), report=True) pdftotext = pbs.Command(PDFTOTEXT_PATH) try: - self.add_property(Property('pdftotext', _(u'pdftotext version'), pdftotext('-v').stderr)) + namespace.add_property('pdftotext', _(u'pdftotext version'), pdftotext('-v').stderr, report=True) except pbs.CommandNotFound: - self.add_property(Property('pdftotext', _(u'pdftotext version'), _(u'not found'))) + namespace.add_property('pdftotext', _(u'pdftotext version'), _(u'not found'), report=True) except Exception: - self.add_property(Property('pdftotext', _(u'pdftotext version'), _(u'error getting version'))) + namespace.add_property('pdftotext', _(u'pdftotext version'), _(u'error getting version'), report=True) - self.add_property(Property('mayan_version', _(u'Mayan EDMS version'), mayan_version)) - self.add_property(Property('fabfile', _(u'Installed via fabfile'), os.path.exists(FABFILE_MARKER))) + def mayan_properties(self): + namespace = PropertyNamespace('mayan', _(u'Mayan EDMS')) + + namespace.add_property('uuid', _(u'UUID'), self.uuid, report=True) + namespace.add_property('mayan_version', _(u'Mayan EDMS version'), mayan_version, report=True) + namespace.add_property('fabfile', _(u'Installed via fabfile'), os.path.exists(FABFILE_MARKER), report=True) + + def git_properties(self): + namespace = PropertyNamespace('git', _(u'Git repository')) try: repo = Repo(settings.PROJECT_ROOT) except: - self.add_property(Property('is_git_repo', _(u'Running from a Git repository'), False)) + namespace.add_property(Property('is_git_repo', _(u'Running from a Git repository'), False)) else: repo.config_reader() headcommit = repo.head.commit - self.add_property(Property('is_git_repo', _(u'Running from a Git repository'), True)) - self.add_property(Property('repo_remotes', _(u'Repository remotes'), ', '.join([unicode(remote) for remote in repo.remotes]))) - self.add_property(Property('repo_remotes_urls', _(u'Repository remotes URLs'), ', '.join([unicode(remote.url) for remote in repo.remotes]))) - self.add_property(Property('repo_head_reference', _(u'Branch'), repo.head.reference)) - self.add_property(Property('headcommit_hexsha', _(u'HEAD commit hex SHA'), headcommit.hexsha)) - self.add_property(Property('headcommit_author', _(u'HEAD commit author'), headcommit.author)) - self.add_property(Property('headcommit_authored_date', _(u'HEAD commit authored date'), time.asctime(time.gmtime(headcommit.authored_date)))) - self.add_property(Property('headcommit_committer', _(u'HEAD commit committer'), headcommit.committer)) - self.add_property(Property('headcommit_committed_date', _(u'HEAD commit committed date'), time.asctime(time.gmtime(headcommit.committed_date)))) - self.add_property(Property('headcommit_message', _(u'HEAD commit message'), headcommit.message)) + namespace.add_property('is_git_repo', _(u'Running from a Git repository'), True) + namespace.add_property('repo_remotes', _(u'Repository remotes'), ', '.join([unicode(remote) for remote in repo.remotes]), report=True) + namespace.add_property('repo_remotes_urls', _(u'Repository remotes URLs'), ', '.join([unicode(remote.url) for remote in repo.remotes]), report=True) + namespace.add_property('repo_head_reference', _(u'Branch'), repo.head.reference, report=True) + namespace.add_property('headcommit_hexsha', _(u'HEAD commit hex SHA'), headcommit.hexsha, report=True) + namespace.add_property('headcommit_author', _(u'HEAD commit author'), headcommit.author) + namespace.add_property('headcommit_authored_date', _(u'HEAD commit authored date'), time.asctime(time.gmtime(headcommit.authored_date)), report=True) + namespace.add_property('headcommit_committer', _(u'HEAD commit committer'), headcommit.committer) + namespace.add_property('headcommit_committed_date', _(u'HEAD commit committed date'), time.asctime(time.gmtime(headcommit.committed_date)), report=True) + namespace.add_property('headcommit_message', _(u'HEAD commit message'), headcommit.message, report=True) + + def set_properties(self): + self._properties = SortedDict() + self.os_properties() + self.binary_dependencies() + self.mayan_properties() + self.git_properties() def __getattr__(self, name): self.set_properties() @@ -142,47 +149,10 @@ class Installation(Singleton): except LockError: pass else: - try: - dictionary = {} - if self.is_lsb: - dictionary.update( - { - 'is_lsb': unicode(self.is_lsb), - 'distributor_id': unicode(self.distributor_id), - 'description': unicode(self.description), - 'release': unicode(self.release), - 'codename': unicode(self.codename), - 'sysinfo': unicode(self.sysinfo), - } - ) + self.set_properties() - dictionary.update( - { - 'uuid': self.uuid, - 'architecture': unicode(self.architecture), - 'python_version': unicode(self.python_version), - 'platform': unicode(self.platform), - 'machine': unicode(self.machine), - 'processor': unicode(self.processor), - 'cpus': unicode(self.cpus), - 'total_phymem': unicode(self.total_phymem), - 'mayan_version': unicode(self.mayan_version), - 'fabfile': unicode(self.fabfile), - } - ) - if self.is_git_repo: - dictionary.update( - { - 'repo_remotes': unicode(self.repo_remotes), - 'repo_remotes_urls': unicode(self.repo_remotes_urls), - 'repo_head_reference': unicode(self.repo_head_reference), - 'headcommit_hexsha': unicode(self.headcommit_hexsha), - 'headcommit_authored_date': unicode(self.headcommit_authored_date), - 'headcommit_committed_date': unicode(self.headcommit_committed_date), - 'headcommit_message': unicode(self.headcommit_message), - } - ) - requests.post(FORM_SUBMIT_URL, data={'formkey': FORM_KEY, FORM_RECEIVER_FIELD: dumps(dictionary)}, timeout=TIMEOUT) + try: + requests.post(FORM_SUBMIT_URL, data={'formkey': FORM_KEY, FORM_RECEIVER_FIELD: Property.get_reportable(as_json=True)}, timeout=TIMEOUT) except (requests.exceptions.Timeout, requests.exceptions.ConnectionError): pass else: From 4b8dc31d237d1ec063f6c9b1fba27917d11315a0 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 25 Jan 2013 22:01:30 -0400 Subject: [PATCH 2/3] Create the views one for the installation properties namespaces and another for the details of each namespace --- apps/installation/__init__.py | 41 ++++++++++++++++++++++++++++++----- apps/installation/links.py | 4 +++- apps/installation/urls.py | 3 ++- apps/installation/views.py | 36 +++++++++++++++++++++--------- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/apps/installation/__init__.py b/apps/installation/__init__.py index 3526291435..160386767e 100644 --- a/apps/installation/__init__.py +++ b/apps/installation/__init__.py @@ -1,15 +1,19 @@ from __future__ import absolute_import -from south.signals import post_migrate - -from project_tools.api import register_tool - from django.db import transaction from django.db.models.signals import post_save from django.db.utils import DatabaseError from django.dispatch import receiver +from django.utils.translation import ugettext_lazy as _ -from .links import installation_details +from south.signals import post_migrate + +from common.utils import encapsulate +from navigation.api import register_links, register_model_list_columns +from project_tools.api import register_tool + +from .classes import Property, PropertyNamespace +from .links import link_menu_link, link_namespace_details, link_namespace_list from .models import Installation @@ -33,6 +37,31 @@ def check_first_run(): details.submit() -register_tool(installation_details) +register_model_list_columns(PropertyNamespace, [ + { + 'name': _(u'label'), + 'attribute': 'label' + }, + { + 'name': _(u'items'), + 'attribute': encapsulate(lambda entry: len(entry.get_properties())) + } +]) + +register_model_list_columns(Property, [ + { + 'name': _(u'label'), + 'attribute': 'label' + }, + { + 'name': _(u'value'), + 'attribute': 'value' + } +]) + +register_links(PropertyNamespace, [link_namespace_details]) +register_links(['namespace_list', PropertyNamespace], [link_namespace_list], menu_name='secondary_menu') + +register_tool(link_menu_link) check_first_run() diff --git a/apps/installation/links.py b/apps/installation/links.py index b1a9445deb..42dac720f3 100644 --- a/apps/installation/links.py +++ b/apps/installation/links.py @@ -4,4 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from .permissions import PERMISSION_INSTALLATION_DETAILS -installation_details = {'text': _(u'installation details'), 'view': 'installation_details', 'icon': 'interface_preferences.png', 'permissions': [PERMISSION_INSTALLATION_DETAILS]} +link_menu_link = {'text': _(u'installation details'), 'view': 'namespace_list', 'icon': 'interface_preferences.png', 'permissions': [PERMISSION_INSTALLATION_DETAILS]} +link_namespace_list = {'text': _(u'installation property namespaces'), 'view': 'namespace_list', 'famfam': 'layout', 'permissions': [PERMISSION_INSTALLATION_DETAILS]} +link_namespace_details = {'text': _(u'details'), 'view': 'namespace_details', 'args': 'object.id', 'famfam': 'layout_link', 'permissions': [PERMISSION_INSTALLATION_DETAILS]} diff --git a/apps/installation/urls.py b/apps/installation/urls.py index 1d8add27f5..785df3f424 100644 --- a/apps/installation/urls.py +++ b/apps/installation/urls.py @@ -1,5 +1,6 @@ from django.conf.urls.defaults import patterns, url urlpatterns = patterns('installation.views', - url(r'^details/$', 'installation_details', (), 'installation_details'), + url(r'^$', 'namespace_list', (), 'namespace_list'), + url(r'^(?P\w+)/details/$', 'namespace_details', (), 'namespace_details'), ) diff --git a/apps/installation/views.py b/apps/installation/views.py index ac6bd66713..4f9f17bf8b 100644 --- a/apps/installation/views.py +++ b/apps/installation/views.py @@ -7,19 +7,35 @@ from django.core.exceptions import PermissionDenied from permissions.models import Permission +from .classes import Property, PropertyNamespace from .permissions import PERMISSION_INSTALLATION_DETAILS from .models import Installation -def installation_details(request): +def namespace_list(request): Permission.objects.check_permissions(request.user, [PERMISSION_INSTALLATION_DETAILS]) - paragraphs = [] - - for instance in Installation().get_properties(): - paragraphs.append('%s: %s' % (unicode(instance.label), instance.value)) - - return render_to_response('generic_template.html', { - 'paragraphs': paragraphs, - 'title': _(u'Installation environment details') - }, context_instance=RequestContext(request)) + Installation().get_properties() + + return render_to_response('generic_list.html', { + 'object_list': PropertyNamespace.get_all(), + 'title': _(u'installation property namespaces'), + 'hide_object': True, + }, context_instance=RequestContext(request)) + + +def namespace_details(request, namespace_id): + Permission.objects.check_permissions(request.user, [PERMISSION_INSTALLATION_DETAILS]) + + Installation().get_properties() + + namespace = PropertyNamespace.get(namespace_id) + object_list = namespace.get_properties() + title = _(u'installation namespace details for: %s') % namespace.label + + return render_to_response('generic_list.html', { + 'object_list': object_list, + 'hide_object': True, + 'title': title, + 'object': namespace, + }, context_instance=RequestContext(request)) From ceea0ed628b2001464319f5c5d8e36f862a73f2a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 25 Jan 2013 22:45:30 -0400 Subject: [PATCH 3/3] VirtualEnv properties to the installation details report --- apps/installation/classes.py | 91 ++++++++++++++++++++++++++++++++++++ apps/installation/models.py | 13 +++++- 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/apps/installation/classes.py b/apps/installation/classes.py index bbc0df076f..603bcd28b0 100644 --- a/apps/installation/classes.py +++ b/apps/installation/classes.py @@ -1,6 +1,25 @@ +from __future__ import absolute_import + +from collections import namedtuple +import os +import sys + +import pbs + +try: + from pbs import pip + PIP = True +except pbs.CommandNotFound: + PIP = False + +from django.conf import settings from django.utils.simplejson import dumps +class PIPNotFound(Exception): + pass + + class PropertyNamespace(object): _registry = {} @@ -73,3 +92,75 @@ class Property(object): def __str__(self): return str(self.value) + + +Dependency = namedtuple('Dependency', 'name, version, standard') + + +class VirtualEnv(object): + def extract_dependency(self, string): + string = str(string.strip()) + + try: + package, version = string.split('==') + except ValueError: + # item is not installed from package, svn/git maybe + try: + version, package = string.split('=') + except: + # has no version number + return Dependency(string, version=None, standard=True) + else: + version = version.split('#')[0].split(' ')[1] # Get rid of '#egg' and '-e' + return Dependency(package, version, standard=False) + else: + return Dependency(package, version, standard=True) + + + def get_packages_info(self, requirements_file=None): + if requirements_file: + with open(requirements_file) as file_in: + for line in file_in.readlines(): + yield self.extract_dependency(line) + else: + for item in pip('freeze').splitlines(): + yield self.extract_dependency(item) + + + def __init__(self): + self.requirements_file_path = os.path.join(settings.PROJECT_ROOT, 'requirements', 'production.txt') + if not PIP: + raise PIPNotFound + + + def get_results(self): + requirements = {} + installed_packages = {} + + for item in self.get_packages_info(self.requirements_file_path): + requirements[item.name] = item + + for item in self.get_packages_info(): + installed_packages[item.name] = item + + for name, item in requirements.items(): + try: + if item.standard: + if item.version: + if item.version == installed_packages[name].version: + status = item.version + else: + status = installed_packages[name].version + else: + status = None + else: + # Non standard version number, check SVN or GIT path + if item.version == installed_packages['%s-dev' % name.replace('-', '_')].version: + status = item.version + else: + status = installed_packages['%s-dev' % name.replace('-', '_')].version + except KeyError: + # Not installed package found matching with name matchin requirement + status = False + + yield name, item.version, status diff --git a/apps/installation/models.py b/apps/installation/models.py index 1779b83c14..d05f698eed 100644 --- a/apps/installation/models.py +++ b/apps/installation/models.py @@ -30,7 +30,7 @@ from main import __version__ as mayan_version from lock_manager import Lock, LockError from ocr.conf.settings import TESSERACT_PATH, UNPAPER_PATH, PDFTOTEXT_PATH -from .classes import Property, PropertyNamespace +from .classes import Property, PropertyNamespace, VirtualEnv, PIPNotFound FORM_SUBMIT_URL = 'https://docs.google.com/spreadsheet/formResponse' FORM_KEY = 'dGZrYkw3SDl5OENMTG15emp1UFFEUWc6MQ' @@ -129,12 +129,23 @@ class Installation(Singleton): namespace.add_property('headcommit_committed_date', _(u'HEAD commit committed date'), time.asctime(time.gmtime(headcommit.committed_date)), report=True) namespace.add_property('headcommit_message', _(u'HEAD commit message'), headcommit.message, report=True) + def virtualenv_properties(self): + namespace = PropertyNamespace('venv', _(u'VirtuanEnv')) + try: + venv = VirtualEnv() + except PIPNotFound: + namespace.add_property('pip', 'pip', _(u'pip not found.'), report=True) + else: + for item, version, result in venv.get_results(): + namespace.add_property(item, '%s (%s)' % (item, version), result, report=True) + def set_properties(self): self._properties = SortedDict() self.os_properties() self.binary_dependencies() self.mayan_properties() self.git_properties() + self.virtualenv_properties() def __getattr__(self, name): self.set_properties()