Source code for stoqlib.database.migration

# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4

## Copyright (C) 2007-2012 Async Open Source <>
## All rights reserved
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU Lesser General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## GNU Lesser General Public License for more details.
## You should have received a copy of the GNU Lesser General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., or visit:
## Author(s): Stoq Team <>
"""Schema migration

import fnmatch
import logging
import os
import re
import shutil
import sys
import tempfile
import traceback

from kiwi.environ import environ

from stoqlib.database.runtime import get_default_store, new_store
from stoqlib.database.settings import db_settings, check_extensions
from stoqlib.domain.plugin import InstalledPlugin
from stoqlib.domain.profile import update_profile_applications
from stoqlib.exceptions import (DatabaseInconsistency, StoqlibError,
from stoqlib.lib.crashreport import collect_traceback
from stoqlib.lib.defaults import stoqlib_gettext
from stoqlib.lib.message import error, info
from stoqlib.lib.parameters import sysparam
from stoqlib.lib.pluginmanager import get_plugin_manager

_ = stoqlib_gettext
log = logging.getLogger(__name__)
# Used by the wizard
create_log = logging.getLogger('stoqlib.database.create')

[docs]class Patch(object): """A Database Patch :attribute filename: patch filename :attribute level: database level """ def __init__(self, filename, migration): """ Create a new Patch object. :param filename: :param migration """ self.filename = filename # Base is the part of the filename minus the extension base = os.path.basename(filename).split('.')[0] # "patch-00-20" -> (00, 20): (generation, level) base_parts = base.split('-', 2) self.generation = int(base_parts[1]) self.level = int(base_parts[2]) self._migration = migration def __cmp__(self, other): return cmp(self.get_version(), other.get_version())
[docs] def apply(self, store): """Apply the patch :param store: a store """ # Dont lock the database here, since StoqlibSchemaMigration.update has # already did that before starting to apply the patches # SQL statement to update the system_table sql = self._migration.generate_sql_for_patch(self) if self.filename.endswith('.sql'): # Create a temporary file used for writing SQL statements temporary = tempfile.mktemp(prefix="patch-%d-%d-" % self.get_version()) # Overwrite the temporary file with the sql patch we want to apply shutil.copy(self.filename, temporary) # After successfully executing the SQL statements, we need to # make sure that the system_table is updated with the correct # schema generation and patchlevel open(temporary, 'a').write(sql) retcode = db_settings.execute_sql(temporary) if retcode != 0: error('Failed to apply %s, psql returned error code: %d' % ( os.path.basename(self.filename), retcode)) os.unlink(temporary) elif self.filename.endswith('.py'): # Execute the patch, we cannot use __import__() since there are # hyphens in the filename and data/sql lacks an ns = {} execfile(self.filename, ns, ns) function = ns['apply_patch'] # Create a new store that will be used to apply the patch and # to update the system tables after the patch has been successfully # applied patch_store = new_store() # Apply the patch itself function(patch_store) # After applying the patch, update the system_table within the same # transaction patch_store.execute(sql) patch_store.commit(close=True) else: raise AssertionError("Unknown filename: %s" % (self.filename, ))
[docs] def get_version(self): """Returns the patch version :returns: a tuple with the patch generation and level """ return self.generation, self.level
[docs]class SchemaMigration(object): """Schema migration management Is currently doing the following things: - Applies database patches - Makes sure that all parameters are present - Makes sure that all applications are present """ patch_resource_domain = None patch_resource = None patch_patterns = ["patch*.sql", "patch*.py"] def __init__(self): if self.patch_resource is None: raise ValueError( _("%s needs to have the patch_resource class variable set") % ( self.__class__.__name__)) if self.patch_patterns is None: raise ValueError( _("%s needs to have the patch_patterns class variable set") % ( self.__class__.__name__)) self.default_store = get_default_store() try: check_extensions(store=self.default_store) except ValueError: error("Missing PostgreSQL extension on the server, " "please install postgresql-contrib") def _patchname_is_valid(self, filename): # simple checking of the patch naming convention valid_patterns = [r"patch-\d\d-\d\d.sql", r"patch-\d\d-\d\"] for valid_pattern in valid_patterns: if re.match(valid_pattern, filename) is not None: return True return False def _get_patches(self): patches = [] for filename in environ.get_resource_names(self.patch_resource_domain, self.patch_resource): for pattern in self.patch_patterns: if not fnmatch.fnmatch(filename, pattern): continue if not self._patchname_is_valid(filename): print("Invalid patch name: %s" % filename) continue filename = environ.get_resource_filename( self.patch_resource_domain, self.patch_resource, filename) patches.append(Patch(filename, self)) return sorted(patches) def _update_schema(self): """Check the current version of database and update the schema if it's needed """"Updating schema") if self.check_uptodate():"Schema is already up to date") return patches = self._get_patches() latest_available = patches[-1].get_version() current_version = self.get_current_version() last_level = None if current_version != latest_available: patches_to_apply = [] for patch in patches: if patch.get_version() <= current_version: continue patches_to_apply.append(patch) from stoqlib.database.admin import create_database_functions create_database_functions()"Applying %d patches" % (len(patches_to_apply), ))"PATCHES:%d" % (len(patches_to_apply), )) for patch in patches_to_apply:"PATCH:%d.%d" % (patch.generation, patch.level)) patch.apply(self.default_store) assert patches_to_apply"All patches (%s) applied." % ( ', '.join(str(p.level) for p in patches_to_apply))) last_level = patches_to_apply[-1].get_version() self.after_update() return current_version, last_level # Public API
[docs] def check(self, check_plugins=True): if self.check_uptodate(): return True if not check_plugins: return True if self.check_plugins(): return True error(_("Database schema error"), _("The database schema has changed, but the database has " "not been updated. Run 'stoqdbadmin updateschema` to " "update the schema to the latest available version.")) return False
[docs] def check_uptodate(self): """ Verify if the schema is up to date. :returns: True or False. """ # Fetch the latest, eg the last in the list patches = self._get_patches() latest_available = patches[-1].get_version() current_version = self.get_current_version() if current_version == latest_available: return True elif current_version > latest_available: current = "(%d.%d)" % current_version latest = "(%d.%d)" % latest_available raise DatabaseInconsistency( _('The current version of database %s is greater than the ' 'latest available version %s. Try upgrading your ' 'installation.') % (current, latest)) return False
def _log(self, msg):
[docs] def apply_all_patches(self): """Apply all available patches """"Applying all patches") current_version = self.get_current_version() to_apply = [] for patch in self._get_patches(): if patch.get_version() > current_version: to_apply.append(patch) self._log("PATCHES:%d" % (len(to_apply), )) for i, patch in enumerate(to_apply): self._log("PATCH:%d" % (i, )) patch.apply(self.default_store) self._log("PATCHES APPLIED")
[docs] def update(self): """Updates the database schema """ if self.check_uptodate(): print('Database is already at the latest version %d.%d' % ( self.get_current_version())) else: from_, to = self._update_schema() if to is None: print('Database schema is already up to date') else: f = "(%d.%d)" % from_ t = "(%d.%d)" % to print('Database schema updated from %s to %s' % (f, t))
[docs] def get_current_version(self): """This method is revision for returning the database schema version for a migration subclass This must be implemented in a subclass :returns: the current database patch version """ raise NotImplementedError
[docs] def generate_sql_for_patch(self, patch): """This method is responsible for creating an SQL statement which is used to update the migration versioning information This must be implemented in a subclass :param patch: the patch that was applied :returns: an SQL string """ raise NotImplementedError
[docs] def after_update(self): """This can be implemented in a subclass, but it is not mandatory. It'll be called after applying all patches """
[docs]class StoqlibSchemaMigration(SchemaMigration): """This is a SchemaMigration subclass used by Stoqlib. It's responsible for migrating the data for stoqlib itself and all its plugins """ patch_resource_domain = 'stoq' patch_resource = 'sql' def __init__(self): super(StoqlibSchemaMigration, self).__init__() self._backup = None def _check_database(self): try:"Locking database") self.default_store.lock_database() except DatabaseError: msg = _('Could not lock database. This means there are other clients ' 'connected. Make sure to close every Stoq client ' 'before updating the database') error(msg) # Database migration is actually run in subprocesses, We need to unlock # the tables again and let the upgrade continue"Releasing database lock") self.default_store.unlock_database() sucess = db_settings.test_connection() if not sucess: # FIXME: Improve this message after 1.5 is released msg = _(u'Could not connect to the database using command line ' 'tool! Aborting.') + ' ' msg += _(u'Please, check if you can connect to the database ' 'using:') + ' ' msg += _(u'psql -l -h <server> -p <port> -U <username>') error(msg) return return True def _backup_database(self): temporary = tempfile.mktemp(prefix="stoq-dump-")"Making a backup to %s" % (temporary, ))"BACKUP-START:") success = db_settings.dump_database(temporary) if not success: info(_(u'Could not create backup! Aborting.')) info(_(u'Please contact stoq team to inform this problem.\n')) return self._backup = temporary return True def _restore_backup(self): if not self._backup: return"Restoring backup %s" % (self._backup, ))"RESTORE-START:") new_name = db_settings.restore_database(self._backup)"RESTORE-DONE:%s" % (new_name, )) def _remove_backup(self): if not self._backup: return os.unlink(self._backup)
[docs] def update(self, plugins=True, backup=True, check_database=True):"Upgrading database (plugins=%r, backup=%r)" % ( plugins, backup)) if check_database and not self._check_database(): return False if backup: self._backup_database() # Don't try to update the plugins if the database doesn't # have the plugin_egg table, which was included in patch-05-15 if self.get_current_version() >= (5, 15): manager = get_plugin_manager() for egg_plugin in manager.egg_plugins_names: try: manager.download_plugin(egg_plugin) except Exception: pass # We have to wrap a try/except statement inside a try/finally to # support python previous to 2.5 version. try: try: super(StoqlibSchemaMigration, self).update() if plugins: self.update_plugins() except Exception: exc = sys.exc_info() tb_str = ''.join(traceback.format_exception(*exc)) collect_traceback(exc, submit=True)"ERROR:%s" % (tb_str, )) self._restore_backup() return False finally: self._remove_backup()"Migration done") return True
def _get_plugins(self): manager = get_plugin_manager() for plugin_name in manager.installed_plugins_names: if plugin_name in manager.available_plugins_names: yield manager.get_plugin(plugin_name)
[docs] def update_plugins(self): for plugin in self._get_plugins(): migration = plugin.get_migration() if migration: migration.update()
[docs] def check_plugins(self): # This cannot be done in check_uptodate since the plugin domain # classes were introduced as a patch and the way the callsites # works in stoq/lib/ for plugin in self._get_plugins(): migration = plugin.get_migration() if not migration: continue if not migration.check_uptodate(): return False return True
[docs] def get_current_version(self): result = self.default_store.execute( """SELECT generation, patchlevel FROM system_table ORDER BY updated DESC LIMIT 1;""") value = result.get_one() result.close() return value
[docs] def after_update(self): # checks if there is new applications and update all the user # profiles on the system store = new_store() update_profile_applications(store) # Updating the parameter list sysparam.ensure_system_parameters(store, update=True) store.commit(close=True)
[docs] def generate_sql_for_patch(self, patch): return self.default_store.quote_query( "INSERT INTO system_table (updated, patchlevel, generation)" "VALUES (NOW(), %s, %s);", (patch.level, patch.generation))
[docs]class PluginSchemaMigration(SchemaMigration): """This is a SchemaMigration class which is suitable for use within a plugin """ def __init__(self, plugin_name, resource_domain, resource, patterns): """ Create a new PluginSchemaMigration object. :param plugin_name: name of the plugin :param resource: resource to load sql patches from :param patterns: sql patch pattern """ self.plugin_name = plugin_name self.patch_resource_domain = resource_domain self.patch_resource = resource self.patch_patterns = patterns SchemaMigration.__init__(self) self._plugin = self.default_store.find( InstalledPlugin, plugin_name=self.plugin_name).one() def _log(self, msg):'PLUGIN ' + msg)
[docs] def generate_sql_for_patch(self, patch): assert self._plugin return self.default_store.quote_query( "UPDATE installed_plugin " "SET plugin_version = %s " "WHERE id = %s;", (patch.level,
[docs] def get_current_version(self): if self._plugin: return (0, self._plugin.plugin_version) return (0, 0)
[docs]def needs_schema_update(): try: migration = StoqlibSchemaMigration() except StoqlibError: error(_("Update Error"), _("You need to call setup() before checking the database " "schema.")) try: update = not (migration.check_uptodate() and migration.check_plugins()) except DatabaseInconsistency as e: error(str(e)) # If we need to update the database, we need to close the connection, # otherwise the locking of the database will fail, since this connection has # already queried a few tables if update: migration.default_store.commit() return update