Add registry app updates

This commit is contained in:
Roberto Rosario
2012-09-07 02:04:00 -04:00
parent 0baae26168
commit 9e4a7025de
10 changed files with 235 additions and 290 deletions

View File

@@ -1,54 +1,80 @@
from __future__ import absolute_import
import inspect
#import runpy
from django.conf import settings
from django.db import transaction, DatabaseError
from django.utils.translation import ugettext_lazy as _
from django.utils.importlib import import_module
from common.utils import encapsulate
from icons.literals import APP, BACKUPS
from job_processor.exceptions import JobQueuePushError
from job_processor.models import JobQueue, JobType
#from common.utils import encapsulate
#from job_processor.exceptions import JobQueuePushError
#from job_processor.models import JobQueue, JobType
from project_tools.api import register_tool
from project_setup.api import register_setup
from navigation.api import bind_links, register_model_list_columns
#from project_setup.api import register_setup
#from navigation.api import bind_links, register_model_list_columns
from .classes import AppBackup, ModelBackup
from .links import (app_registry_tool_link, app_list, backup_tool_link,
restore_tool_link, backup_job_list, backup_job_create, backup_job_edit,
backup_job_test)
from .literals import BACKUP_JOB_QUEUE_NAME
from .models import App, BackupJob
#from .classes import AppBackup, ModelBackup
#from .links import (app_registry_tool_link, app_list, backup_tool_link,
# restore_tool_link, backup_job_list, backup_job_create, backup_job_edit,
# backup_job_test)
#from .literals import BACKUP_JOB_QUEUE_NAME
from .models import App
#from . import models
#class UnableToRegister(Exception):
# pass
#apipkg.initpkg(__name__, {
# #'App': _App,
## 'App': 'app_registry.models:App',
# #'App': models.App
#})
#pp = 1
#from .models import App#as _App#, BackupJob as _BackupJob
#@transaction.commit_on_success
#def create_backups_job_queue():
# global backups_job_queue
# try:
# backups_job_queue, created = JobQueue.objects.get_or_create(name=BACKUP_JOB_QUEUE_NAME, defaults={'label': _('Backups'), 'unique_jobs': True})
# except DatabaseError:
# transaction.rollback()
@transaction.commit_on_success
def create_backups_job_queue():
global backups_job_queue
#bind_links(['app_list'], [app_list], menu_name='secondary_menu')
#create_backups_job_queue()
###backup_job_type = JobType('remote_backup', _(u'Remove backup'), do_backup)
#register_setup(backup_tool_link)
#register_tool(restore_tool_link)
#bind_links([BackupJob, 'backup_job_list', 'backup_job_create'], [backup_job_list], menu_name='secondary_menu')
#bind_links([BackupJob, 'backup_job_list', 'backup_job_create'], [backup_job_create], menu_name='sidebar')
#bind_links([BackupJob], [backup_job_edit, backup_job_test])
#register_model_list_columns(BackupJob, [
# {'name':_(u'begin date time'), 'attribute': 'begin_datetime'},
# {'name':_(u'storage module'), 'attribute': 'storage_module.label'},
# {'name':_(u'apps'), 'attribute': encapsulate(lambda x: u', '.join([unicode(app) for app in x.apps.all()]))},
#])
###app.set_backup([ModelBackup()])
for app_name in settings.INSTALLED_APPS:
App.register(app_name)
print 'registry', app_name
try:
backups_job_queue, created = JobQueue.objects.get_or_create(name=BACKUP_JOB_QUEUE_NAME, defaults={'label': _('Backups'), 'unique_jobs': True})
except DatabaseError:
transaction.rollback()
register_tool(app_registry_tool_link)
bind_links(['app_list'], [app_list], menu_name='secondary_menu')
create_backups_job_queue()
#backup_job_type = JobType('remote_backup', _(u'Remove backup'), do_backup)
register_setup(backup_tool_link)
register_tool(restore_tool_link)
bind_links([BackupJob, 'backup_job_list', 'backup_job_create'], [backup_job_list], menu_name='secondary_menu')
bind_links([BackupJob, 'backup_job_list', 'backup_job_create'], [backup_job_create], menu_name='sidebar')
bind_links([BackupJob], [backup_job_edit, backup_job_test])
register_model_list_columns(BackupJob, [
{'name':_(u'begin date time'), 'attribute': 'begin_datetime'},
{'name':_(u'storage module'), 'attribute': 'storage_module.label'},
{'name':_(u'apps'), 'attribute': encapsulate(lambda x: u', '.join([unicode(app) for app in x.apps.all()]))},
])
try:
app = App.register('app_registry', label=_(u'App registry'), icon=APP, description=_(u'Holds the app registry and backups functions.'))
except App.UnableToRegister:
pass
else:
app.set_backup([ModelBackup()])
post_init = import_module('%s.post_init' % app_name)
except ImportError:
pass
else:
print 'post', post_init
if post_init:
for name, value in inspect.getmembers(post_init):
if hasattr(value, '__call__') and name.startswith('init'):
value()

View File

@@ -4,9 +4,11 @@ import os
from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.core.management.commands.dumpdata import Command
from django.conf import settings
from django.db import router, DEFAULT_DB_ALIAS
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.utils.importlib import import_module
logger = logging.getLogger(__name__)
@@ -156,145 +158,4 @@ class AppBackup(object):
results.append(u'%s - %s' % (manager, manager.info() or _(u'Nothing')))
return u', '.join(results)
def backup(self, storage_module, dry_run=False):
logger.debug('starting')
self.state = self.__class__.STATE_BACKING_UP
for manager in self.backup_managers:
result = manager.backup()
storage_module.backup(result, dry_run=dry_run)
self.state = self.__class__.STATE_IDLE
def restore(self, storage_module=None):
logger.debug('starting')
self.state = self.__class__.STATE_RESTORING
for manager in self.backup_managers:
manager.restore(storage_module.restore())
self.state = self.__class__.STATE_IDLE
def __unicode__(self):
return unicode(self.app)
#Storage
class StorageModuleBase(object):
_registry = {}
# Local modules depend on hardware on a node and execute in the Scheduler
# of a particular node
REALM_LOCAL = 'local'
# Remote modules can be execute by any node in a cluster and are placed
# in the JobQueue
REALM_REMOTE = 'remote'
REALM_CHOICES = (
(REALM_LOCAL, _(u'local')),
(REALM_REMOTE, _(u'remote')),
)
class UnknownStorageModule(Exception):
pass
@classmethod
def register(cls, klass):
"""
Register a subclass of StorageModuleBase to make it available to the
UI
"""
cls._registry[klass.name] = klass
@classmethod
def get_all(cls):
return cls._registry.values()
@classmethod
def get(cls, name):
try:
return cls._registry[name]
except KeyError:
raise cls.UnknownStorageModule
@classmethod
def get_as_choices(cls):
return [(name, unicode(klass.label)) for name, klass in cls._registry.items()]
def get_arguments(self):
return []
def is_local_realm(self):
return self.realm == REALM_LOCAL
def is_remote_realm(self):
return self.realm == REALM_REMOTE
def backup(self, data, dry_run):
raise NotImplemented
def restore(self):
"""
Must return data or a file like object
"""
raise NotImplemented
def __unicode__(self):
return unicode(self.label)
class TestStorage(StorageModuleBase):
name = 'test_storage'
label = _(u'Test storage module')
realm = StorageModuleBase.REALM_LOCAL
def __init__(self, *args, **kwargs):
self.backup_path = kwargs.pop('backup_path', None)
self.restore_path = kwargs.pop('restore_path', None)
def get_arguments(self):
return ['backup_path', 'restore_path']
def backup(self, elements, dry_run):
logger.debug('self.backup_path: %s' % self.backup_path)
for element in elements:
content_file = element.save()
logger.debug('element.filename: %s' % element.filename)
logger.debug('element.content: %s' % element.content)
def restore(self):
print 'restore from path: %s' % self.restore_path
return 'sample_data'
class LocalFileSystemStorage(FileSystemStorage):
"""
Simple wrapper for the stock Django FileSystemStorage class
"""
name = 'local_filesystem_storage'
label = _(u'Local filesystem')
realm = StorageModuleBase.REALM_LOCAL
separator = os.path.sep
def get_arguments(self):
return ['backup_path', 'restore_path']
def backup(self, elements, dry_run):
logger.debug('self.backup_path: %s' % self.backup_path)
for element in elements:
content_file = element.save()
path = self.storage.save(content_file.name, content_file)
logger.debug('element.filename: %s' % element.filename)
logger.debug('element.content: %s' % element.content)
def restore(self):
print 'restore from path: %s' % self.restore_path
return 'sample_data'
def __init__(self, *args, **kwargs):
self.backup_path = kwargs.pop('backup_path', None)
self.storage = FileSystemStorage(location=self.backup_path)
StorageModuleBase.register(LocalFileSystemStorage)
StorageModuleBase.register(TestStorage)

View File

@@ -4,17 +4,17 @@ from django import forms
from common.widgets import ScrollableCheckboxSelectMultiple
from .classes import AppBackup
from .models import App, BackupJob
#from .classes import AppBackup
#from .models import App, BackupJob
def valid_app_choices():
# Return app that exist in the app registry and that have been registered for backup
return App.live.filter(pk__in=[appbackup.app.pk for appbackup in AppBackup.get_all()])
#def valid_app_choices():
# # Return app that exist in the app registry and that have been registered for backup
# return App.live.filter(pk__in=[appbackup.app.pk for appbackup in AppBackup.get_all()])
class BackupJobForm(forms.ModelForm):
apps = forms.ModelMultipleChoiceField(queryset=valid_app_choices(), widget=ScrollableCheckboxSelectMultiple())
class Meta:
model = BackupJob
#class BackupJobForm(forms.ModelForm):
# apps = forms.ModelMultipleChoiceField(queryset=valid_app_choices(), widget=ScrollableCheckboxSelectMultiple())
#
# class Meta:
# model = BackupJob

View File

@@ -0,0 +1,6 @@
from __future__ import absolute_import
from icons.literals import PLUGIN
from icons import Icon
icon_app = Icon(PLUGIN)

View File

@@ -3,13 +3,12 @@ from __future__ import absolute_import
from django.utils.translation import ugettext_lazy as _
from navigation.api import Link
from icons.api import get_icon_name, get_sprite_name
from icons.literals import APP
from .icons import icon_app
from .permissions import PERMISSION_BACKUP_JOB_VIEW, PERMISSION_BACKUP_JOB_CREATE, PERMISSION_BACKUP_JOB_EDIT, PERMISSION_BACKUP_JOB_DELETE
app_registry_tool_link = Link(text=_(u'Apps'), view='app_list', icon=get_icon_name(APP))#, permissions=[PERMISSION_BACKUP_JOB_VIEW])
app_list = Link(text=_(u'app list'), view='app_list', sprite=get_sprite_name(APP))#, permissions=[PERMISSION_BACKUP_JOB_VIEW])
app_registry_tool_link = Link(text=_(u'Apps'), view='app_list', icon=icon_app)#, permissions=[PERMISSION_BACKUP_JOB_VIEW])
app_list = Link(text=_(u'app list'), view='app_list', sprite=icon_app)#, permissions=[PERMISSION_BACKUP_JOB_VIEW])
backup_tool_link = Link(text=_(u'backups'), view='backup_job_list', icon='cd_burn.png', permissions=[PERMISSION_BACKUP_JOB_VIEW])
backup_job_list = Link(text=_(u'backup job list'), view='backup_job_list', sprite='cd_burn', permissions=[PERMISSION_BACKUP_JOB_VIEW])

View File

@@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Deleting model 'BackupJob'
db.delete_table('app_registry_backupjob')
# Removing M2M table for field apps on 'BackupJob'
db.delete_table('app_registry_backupjob_apps')
# Deleting field 'App.icon'
db.delete_column('app_registry_app', 'icon')
def backwards(self, orm):
# Adding model 'BackupJob'
db.create_table('app_registry_backupjob', (
('storage_arguments_json', self.gf('django.db.models.fields.TextField')(blank=True)),
('name', self.gf('django.db.models.fields.CharField')(max_length=64)),
('storage_module_name', self.gf('django.db.models.fields.CharField')(max_length=32)),
('begin_datetime', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2012, 8, 18, 0, 0))),
('enabled', self.gf('django.db.models.fields.BooleanField')(default=True)),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
))
db.send_create_signal('app_registry', ['BackupJob'])
# Adding M2M table for field apps on 'BackupJob'
db.create_table('app_registry_backupjob_apps', (
('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
('backupjob', models.ForeignKey(orm['app_registry.backupjob'], null=False)),
('app', models.ForeignKey(orm['app_registry.app'], null=False))
))
db.create_unique('app_registry_backupjob_apps', ['backupjob_id', 'app_id'])
# Adding field 'App.icon'
db.add_column('app_registry_app', 'icon',
self.gf('django.db.models.fields.CharField')(default='', max_length=64, blank=True),
keep_default=False)
models = {
'app_registry.app': {
'Meta': {'ordering': "('name',)", 'object_name': 'App'},
'dependencies': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['app_registry.App']", 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
}
}
complete_apps = ['app_registry']

View File

@@ -2,6 +2,8 @@ from __future__ import absolute_import
import datetime
import logging
import imp
import sys
from django.db import models
from django.db import DatabaseError, transaction
@@ -9,22 +11,25 @@ from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.utils.importlib import import_module
from common.models import TranslatableLabelMixin, LiveObjectMixin
from smart_settings import SettingsNamespace
from project_setup.api import register_setup
from project_tools.api import register_tool
from .classes import AppBackup, StorageModuleBase
#from .classes import AppBackup, StorageModuleBase, Setting
logger = logging.getLogger(__name__)
class App(TranslatableLabelMixin, LiveObjectMixin, models.Model):
translatables = ['label', 'description']
translatables = ['label', 'description', 'icon']
class UnableToRegister(Exception):
pass
#class UnableToRegister(Exception):
# pass
name = models.CharField(max_length=64, verbose_name=_(u'name'), unique=True)
icon = models.CharField(max_length=64, verbose_name=_(u'icon'), blank=True)
dependencies = models.ManyToManyField('self', verbose_name=_(u'dependencies'), symmetrical=False, blank=True, null=True)
#version
#top_urls
@@ -32,30 +37,62 @@ class App(TranslatableLabelMixin, LiveObjectMixin, models.Model):
@classmethod
@transaction.commit_on_success
def register(cls, name, label, icon=None, description=None):
def register(cls, app_name):
try:
app, created = App.objects.get_or_create(name=name)
except DatabaseError:
transaction.rollback()
raise cls.UnableToRegister
app_module = import_module(app_name)
except ImportError:
transaction.rollback
else:
app.label = label
if icon:
app.icon = icon
if description:
app.description = description
app.dependencies.clear()
app.save()
return app
def set_dependencies(self, app_names):
for app_name in app_names:
app = App.objects.get(name=app_name)
self.dependencies.add(app)
def set_backup(self, *args, **kwargs):
return AppBackup(self, *args, **kwargs)
try:
registration = import_module('%s.registry' % app_name)
except ImportError:
transaction.rollback
else:
disabled = getattr(registration, 'disabled', False)
name = getattr(registration, 'name')
label = getattr(registration, 'label')
icon = getattr(registration, 'icon', None)
description = getattr(registration, 'description', None)
dependencies = getattr(registration, 'dependencies', [])
settings = getattr(registration, 'settings', None)
setup_links = getattr(registration, 'setup_links', [])
tool_links = getattr(registration, 'tool_links', [])
if not disabled:
try:
app, created = App.objects.get_or_create(name=name)
except DatabaseError:
transaction.rollback()
raise cls.UnableToRegister
else:
app.label = label
if description:
app.description = description
app.dependencies.clear()
app.save()
app.icon = icon
for app_name in dependencies:
dependency = App.objects.get(name=app_name)
app.dependencies.add(dependency)
if settings:
settings_module = imp.new_module('settings')
setattr(app_module, 'settings', settings_module)
sys.modules['%s.settings' % name] = settings_module
settings_namespace = SettingsNamespace(name, label, '%s.settings' % name)
for setting in settings:
settings_namespace.add_setting(**setting)
for link in setup_links:
register_setup(link)
for link in tool_links:
register_tool(link)
#def set_backup(self, *args, **kwargs):
# return AppBackup(self, *args, **kwargs)
def __unicode__(self):
return unicode(self.label)
@@ -65,55 +102,3 @@ class App(TranslatableLabelMixin, LiveObjectMixin, models.Model):
verbose_name_plural = _(u'apps')
class BackupJob(models.Model):
name = models.CharField(max_length=64, verbose_name=_(u'name'))
enabled = models.BooleanField(default=True, verbose_name=_(u'enabled'))
apps = models.ManyToManyField(App)
begin_datetime = models.DateTimeField(verbose_name=_(u'begin date and time'), default=lambda: datetime.datetime.now())
# * repetition =
# day - 1 days
# weekly - days of week checkbox
# month - day of month, day of week
# * repetition option field
# * ends
# - never
# - After # ocurrences
# - On date
# * end option field
# * type
# - Full
# - Incremental
storage_module_name = models.CharField(max_length=32, choices=StorageModuleBase.get_as_choices(), verbose_name=_(u'storage module'))
storage_arguments_json = models.TextField(verbose_name=_(u'storage module arguments (in JSON)'), blank=True)
def __unicode__(self):
return self.name
@property
def storage_module(self):
return StorageModuleBase.get(self.storage_module_name)
def backup(self, dry_run=False):
logger.debug('starting: %s', self)
logger.debug('dry_run: %s' % dry_run)
storage_module = self.storage_module
#TODO: loads
for app in self.apps.all():
app_backup = AppBackup.get(app)
app_backup.backup(storage_module(backup_path='/tmp'), dry_run=dry_run)
def save(self, *args, **kwargs):
#dump
super(BackupJob, self).save(*args, **kwargs)
@models.permalink
def get_absolute_url(self):
return ('checkout_info', [self.document.pk])
class Meta:
verbose_name = _(u'document checkout')
verbose_name_plural = _(u'document checkouts')
#class BackupJobLog

View File

@@ -0,0 +1,12 @@
from __future__ import absolute_import
from django.utils.translation import ugettext_lazy as _
from .icons import icon_app
from .links import app_registry_tool_link
name = 'app_registry'
label = _(u'App registry')
description = _(u'Handles the registration of apps in a project.')
icon = icon_app
tool_links = [app_registry_tool_link]

View File

@@ -2,9 +2,9 @@ from django.conf.urls.defaults import patterns, url
urlpatterns = patterns('app_registry.views',
url(r'^list/$', 'app_list', (), 'app_list'),
url(r'^jobs/list/$', 'backup_job_list', (), 'backup_job_list'),
url(r'^jobs/create/$', 'backup_job_create', (), 'backup_job_create'),
url(r'^jobs/(?P<backup_job_pk>\d+)/edit/$', 'backup_job_edit', (), 'backup_job_edit'),
url(r'^jobs/(?P<backup_job_pk>\d+)/test/$', 'backup_job_test', (), 'backup_job_test'),
#url(r'^jobs/list/$', 'backup_job_list', (), 'backup_job_list'),
#url(r'^jobs/create/$', 'backup_job_create', (), 'backup_job_create'),
#url(r'^jobs/(?P<backup_job_pk>\d+)/edit/$', 'backup_job_edit', (), 'backup_job_edit'),
#url(r'^jobs/(?P<backup_job_pk>\d+)/test/$', 'backup_job_test', (), 'backup_job_test'),
#url(r'^jobs/(?P<backup_job_pk>\d+)/delete/$', 'backup_job_delete', (), 'backup_job_delete'),
)

View File

@@ -9,14 +9,13 @@ from django.template import RequestContext
from django.utils.translation import ugettext as _
from common.utils import encapsulate
from icons.widgets import icon_widget
from icons.literals import APP
from permissions.models import Permission
from .classes import AppBackup
from .forms import BackupJobForm
from .models import App, BackupJob
#from .classes import AppBackup
#from .forms import BackupJobForm
from .models import App#, BackupJob
from .permissions import PERMISSION_BACKUP_JOB_VIEW, PERMISSION_BACKUP_JOB_CREATE, PERMISSION_BACKUP_JOB_EDIT
from .icons import icon_app
def app_list(request):
@@ -25,16 +24,16 @@ def app_list(request):
return render_to_response('generic_list.html', {
'object_list' : App.live.all(),
'hide_object': True,
'title': _(u'registered apps'),
'extra_columns': [
{'name': _(u'icon'), 'attribute': 'icon'},
{'name':_(u'icon'), 'attribute': encapsulate(lambda x: icon_widget(x.icon or APP))},
{'name': _(u'label'), 'attribute': 'label'},
{'name':_(u'icon'), 'attribute': encapsulate(lambda x: getattr(x, 'icon', icon_app).display_big())},
{'name':_(u'description'), 'attribute': 'description'},
{'name':_(u'dependencies'), 'attribute': encapsulate(lambda x: u', '.join([unicode(dependency) for dependency in x.dependencies.all()]))},
],
}, context_instance=RequestContext(request))
"""
def backup_job_list(request):
pre_object_list = BackupJob.objects.all()
@@ -142,3 +141,4 @@ def backup_view(request):
}
return render_to_response('generic_list.html', context,
context_instance=RequestContext(request))
"""