diff --git a/fabfile/__init__.py b/fabfile/__init__.py index d4f48fe803..40f2c894ff 100644 --- a/fabfile/__init__.py +++ b/fabfile/__init__.py @@ -1,13 +1,14 @@ +import sys + from fabric.api import task, env from fabric.colors import white import databases as database -import webservers as webserver import platforms as platform +import webservers as webserver import django -from conf import setup_environment, print_supported_configs - -setup_environment() +from conf import print_supported_configs +from server_config import servers print(white('\n\n ######## ', bold=True)) print(white(' ######## ', bold=True)) @@ -27,8 +28,11 @@ print(white('\nMayan EDMS Fabric installation file\n\n', bold=True)) print_supported_configs() -@task(default=True) +@task def install(): + """ + Perform a complete install of Mayan EDMS on a host + """ platform.install_dependencies() platform.install_mayan() platform.install_database_manager() @@ -45,6 +49,9 @@ def install(): @task def uninstall(): + """ + Perform a complete removal of Mayan EDMS from a host + """ platform.delete_mayan() webserver.remove_site() webserver.restart() @@ -52,5 +59,3 @@ def uninstall(): if env.drop_database: database.drop_database() database.drop_username() - - diff --git a/fabfile/conf.py b/fabfile/conf.py index 9574948f9f..16e380234d 100644 --- a/fabfile/conf.py +++ b/fabfile/conf.py @@ -3,12 +3,14 @@ import string import random from fabric.api import env +from fabric.colors import green from literals import (DEFAULT_INSTALL_PATH, DEFAULT_VIRTUALENV_NAME, DEFAULT_REPOSITORY_NAME, DEFAULT_OS, OS_CHOICES, DEFAULT_DATABASE_MANAGER, DB_CHOICES, DEFAULT_DATABASE_NAME, DEFAULT_WEBSERVER, WEB_CHOICES, DEFAULT_DATABASE_USERNAME, DJANGO_DB_DRIVERS, DEFAULT_DATABASE_HOST, DEFAULT_PASSWORD_LENGTH) +from server_config import reduce_env def password_generator(): @@ -16,7 +18,8 @@ def password_generator(): chars = string.ascii_letters + string.digits return ''.join(random.choice(chars) for x in range(DEFAULT_PASSWORD_LENGTH)) - + +@reduce_env def setup_environment(): env['os'] = getattr(env, 'os', DEFAULT_OS) env['os_name'] = OS_CHOICES[env.os] @@ -47,7 +50,11 @@ def setup_environment(): def print_supported_configs(): - print('Supported operating systems (os=): %s' % dict(OS_CHOICES).keys()) - print('Supported database managers (database_manager=): %s' % dict(DB_CHOICES).keys()) - print('Supported webservers (webserver=): %s' % dict(WEB_CHOICES).keys()) + print('Supported operating systems (os=): %s, default=\'%s\'' % (dict(OS_CHOICES).keys(), green(DEFAULT_OS))) + print('Supported database managers (database_manager=): %s, default=\'%s\'' % (dict(DB_CHOICES).keys(), green(DEFAULT_DATABASE_MANAGER))) + print('Supported webservers (webserver=): %s, default=\'%s\'' % (dict(WEB_CHOICES).keys(), green(DEFAULT_WEBSERVER))) print('\n') + + + + diff --git a/fabfile/databases/__init__.py b/fabfile/databases/__init__.py index ad48ada637..8afccd8032 100644 --- a/fabfile/databases/__init__.py +++ b/fabfile/databases/__init__.py @@ -1,7 +1,7 @@ from fabric.api import env, task from fabric.colors import green - +from ..conf import setup_environment from ..literals import DB_MYSQL import mysql @@ -11,6 +11,7 @@ def create_database(): """ Create the Mayan EDMS database """ + setup_environment() print(green('Creating Mayan EDMS database', bold=True)) if env.database_manager == DB_MYSQL: @@ -22,6 +23,7 @@ def drop_database(): """ Drop Mayan EDMS's database """ + setup_environment() print(green('Droping Mayan EDMS database', bold=True)) if env.database_manager == DB_MYSQL: @@ -33,6 +35,7 @@ def drop_username(): """ Drop Mayan EDMS's username """ + setup_environment() print(green('Droping Mayan EDMS username', bold=True)) if env.database_manager == DB_MYSQL: diff --git a/fabfile/django/__init__.py b/fabfile/django/__init__.py index 294d17075f..9ce0309435 100644 --- a/fabfile/django/__init__.py +++ b/fabfile/django/__init__.py @@ -3,18 +3,32 @@ import os from fabric.api import env, task, cd, sudo from fabric.contrib.files import upload_template +from ..conf import setup_environment + @task def syncdb(): + """ + Perform Django's syncdb command + """ + setup_environment() with cd(env.virtualenv_path): sudo('source bin/activate; %(repository_name)s/manage.py syncdb --noinput; %(repository_name)s/manage.py migrate' % (env)) @task def database_config(): + """ + Create a settings_local.py file tailored to the database manager selected + """ + setup_environment() upload_template(filename=os.path.join('fabfile', 'templates', 'settings_local.py'), destination=env.repository_path, context=env, use_sudo=True) @task def collectstatic(): + """ + Perform Django's collectstatic command + """ + setup_environment() with cd(env.virtualenv_path): sudo('source bin/activate; %(repository_name)s/manage.py collectstatic --noinput' % (env)) diff --git a/fabfile/platforms/__init__.py b/fabfile/platforms/__init__.py index 6ccdf8971d..5a8ec61da4 100644 --- a/fabfile/platforms/__init__.py +++ b/fabfile/platforms/__init__.py @@ -2,6 +2,7 @@ from fabric.api import run, sudo, cd, env, task from fabric.colors import green from ..literals import OS_UBUNTU, OS_FEDORA, OS_DEBIAN +from ..conf import setup_environment import linux, ubuntu, fedora, debian @@ -10,7 +11,7 @@ def install_dependencies(): """ Install OS dependencies """ - + setup_environment() print(green('Installing dependencies for %s' % env.os_name, bold=True)) if env.os in [OS_UBUNTU, OS_DEBIAN]: @@ -24,7 +25,7 @@ def install_mayan(): """ Install Mayan EDMS """ - + setup_environment() print(green('Installing Mayan EDMS from git repository', bold=True)) if env.os in [OS_UBUNTU, OS_FEDORA, OS_DEBIAN]: @@ -36,7 +37,7 @@ def install_database_manager(): """ Install the selected database manager """ - + setup_environment() print(green('Installing database manager: %s' % env.database_manager_name, bold=True)) if env.os in [OS_UBUNTU, OS_DEBIAN]: @@ -50,7 +51,7 @@ def fix_permissions(): """ Fix installation files' permissions """ - + setup_environment() print(green('Fixing installation files\' permissions', bold=True)) if env.os in [OS_UBUNTU, OS_DEBIAN]: @@ -64,7 +65,7 @@ def install_webserver(): """ Installing the OS packages for the webserver """ - + setup_environment() print(green('Installing webserver: %s' % env.webserver_name, bold=True)) if env.os in [OS_UBUNTU, OS_DEBIAN]: @@ -78,7 +79,7 @@ def delete_mayan(): """ Delete Mayan EDMS from the OS """ - + setup_environment() print(green('Deleting Mayan EDMS files', bold=True)) if env.os in [OS_UBUNTU, OS_FEDORA, OS_DEBIAN]: @@ -90,6 +91,7 @@ def post_install(): """ Perform post install operations """ + setup_environment() if env.os == OS_UBUNTU: ubuntu.post_install() elif env.os == OS_FEDORA: diff --git a/fabfile/server_config.py b/fabfile/server_config.py new file mode 100644 index 0000000000..29228e4a74 --- /dev/null +++ b/fabfile/server_config.py @@ -0,0 +1,141 @@ +"""Fabric server config management fabfile. +If you need additional configuration, setup ~/.fabricrc file: + + user = your_remote_server_username + +To get specific command help type: + fab -d command_name + +""" +# From http://fueledbylemons.com/blog/2011/04/09/server-configs-and-fabric/ + + +import os + +from fabric.api import env, task +from fabric.utils import puts +from fabric import colors +import fabric.network +import fabric.state + + +YAML_AVAILABLE = True +try: + import yaml +except ImportError: + YAML_AVAILABLE = False + + +JSON_AVAILABLE = True +try: + import simplejson as json +except ImportError: + try: + import json + except ImportError: + JSON_AVAILABLE = False + +################################ +# ENVIRONMENTS # +################################ + +def _load_config(**kwargs): + """Find and parse server config file. + + If `config` keyword argument wasn't set look for default + 'server_config.yaml' or 'server_config.json' file. + + """ + config, ext = os.path.splitext(kwargs.get('config', + 'server_config.yaml' if os.path.exists('server_config.yaml') else 'server_config.json')) + + if not os.path.exists(config + ext): + print colors.red('Error. "%s" file not found.' % (config + ext)) + return {} + if YAML_AVAILABLE and ext == '.yaml': + loader = yaml + elif JSON_AVAILABLE and ext =='.json': + loader = json + else: + print colors.red('Parser package not available') + return {} + # Open file and deserialize settings. + with open(config + ext) as config_file: + return loader.load(config_file) + +@task +def servers(*args, **kwargs): + """Set destination servers or server groups by comma delimited list of names""" + # Load config + servers = _load_config(**kwargs) + # If no arguments were recieved, print a message with a list of available configs. + if not args: + print 'No server name given. Available configs:' + for key in servers: + print colors.green('\t%s' % key) + + # Create `group` - a dictionary, containing copies of configs for selected servers. Server hosts + # are used as dictionary keys, which allows us to connect current command destination host with + # the correct config. This is important, because somewhere along the way fabric messes up the + # hosts order, so simple list index incrementation won't suffice. + env.group = {} + # For each given server name + for name in args: + # Recursive function call to retrieve all server records. If `name` is a group(e.g. `all`) + # - get it's members, iterate through them and create `group` + # record. Else, get fields from `name` server record. + # If requested server is not in the settings dictionary output error message and list all + # available servers. + _build_group(name, servers) + + + # Copy server hosts from `env.group` keys - this gives us a complete list of unique hosts to + # operate on. No host is added twice, so we can safely add overlaping groups. Each added host is + # guaranteed to have a config record in `env.group`. + env.hosts = env.group.keys() + +def _build_group(name, servers): + """Recursively walk through servers dictionary and search for all server records.""" + # We're going to reference server a lot, so we'd better store it. + server = servers.get(name, None) + # If `name` exists in servers dictionary we + if server: + # check whether it's a group by looking for `members` + if isinstance(server, list): + if fabric.state.output['debug']: + puts("%s is a group, getting members" % name) + for item in server: + # and call this function for each of them. + _build_group(item, servers) + # When, finally, we dig through to the standalone server records, we retrieve + # configs and store them in `env.group` + else: + if fabric.state.output['debug']: + puts("%s is a server, filling up env.group" % name) + env.group[server['host']] = server + else: + print colors.red('Error. "%s" config not found. Run `fab servers` to list all available configs' % name) + +def reduce_env(task): + """ + Copies server config settings from `env.group` dictionary to env variable. + + This way, tasks have easier access to server-specific variables: + `env.owner` instead of `env.group[env.host]['owner']` + + """ + def task_with_setup(*args, **kwargs): + # If `s:server` was run before the current command - then we should copy values to + # `env`. Otherwise, hosts were passed through command line with `fab -H host1,host2 + # command` and we skip. + if env.get("group", None): + for key,val in env.group[env.host].items(): + setattr(env, key, val) + if fabric.state.output['debug']: + puts("[env] %s : %s" % (key, val)) + + task(*args, **kwargs) + # Don't keep host connections open, disconnect from each host after each task. + # Function will be available in fabric 1.0 release. + # fabric.network.disconnect_all() + return task_with_setup diff --git a/fabfile/webservers/__init__.py b/fabfile/webservers/__init__.py index ceaeeebf1e..d6084a6fbe 100644 --- a/fabfile/webservers/__init__.py +++ b/fabfile/webservers/__init__.py @@ -1,7 +1,9 @@ from fabric.api import run, sudo, cd, env, task from fabric.colors import green +from ..conf import setup_environment from ..literals import WEB_APACHE + import apache @@ -10,7 +12,7 @@ def install_site(): """ Install Mayan EDMS site in the webserver configuration files """ - + setup_environment() print(green('Adding Mayan EDMS\'s site files to: %s' % env.webserver_name, bold=True)) if env.webserver == WEB_APACHE: @@ -22,6 +24,7 @@ def remove_site(): """ Install Mayan EDMS's site file from the webserver's configuration """ + setup_environment() print(green('Removing Mayan EDMS\'s site file from %s configuration' % env.webserver_name, bold=True)) if env.webserver == WEB_APACHE: @@ -33,6 +36,7 @@ def restart(): """ Restart the webserver """ + setup_environment() print(green('Restarting the web server: %s' % env.webserver_name, bold=True)) if env.webserver == WEB_APACHE: @@ -44,6 +48,7 @@ def reload(): """ Reload webserver configuration files """ + setup_environment() print(green('Reloading the web server configuration files', bold=True)) if env.webserver == WEB_APACHE: