From 54e418184abcc9836e68cba9138a5c3b6fe82733 Mon Sep 17 00:00:00 2001 From: gau1991 Date: Mon, 2 Mar 2015 18:04:26 +0530 Subject: [PATCH 1/7] Added stack migrate command to migrate MySQL/Percona to MariaDB --- ee/cli/plugins/stack.py | 2 + ee/cli/plugins/stack_migrate.py | 99 +++++++++++++++++++++++++++++++++ ee/core/mysql.py | 43 +++++++++++++- 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 ee/cli/plugins/stack_migrate.py diff --git a/ee/cli/plugins/stack.py b/ee/cli/plugins/stack.py index e0a24349..54a6fbb6 100644 --- a/ee/cli/plugins/stack.py +++ b/ee/cli/plugins/stack.py @@ -26,6 +26,7 @@ import pwd import grp import codecs from ee.cli.plugins.stack_services import EEStackStatusController +from ee.cli.plugins.stack_migrate import EEStackMigrateController from ee.core.logging import Log @@ -1523,6 +1524,7 @@ def load(app): # register the plugin class.. this only happens if the plugin is enabled handler.register(EEStackController) handler.register(EEStackStatusController) + handler.register(EEStackMigrateController) # register a hook (function) to run after arguments are parsed. hook.register('post_argument_parsing', ee_stack_hook) diff --git a/ee/cli/plugins/stack_migrate.py b/ee/cli/plugins/stack_migrate.py new file mode 100644 index 00000000..2e4becf4 --- /dev/null +++ b/ee/cli/plugins/stack_migrate.py @@ -0,0 +1,99 @@ +from cement.core.controller import CementBaseController, expose +from cement.core import handler, hook +from ee.core.mysql import EEMysql +from ee.core.logging import Log +from ee.core.variables import EEVariables +from ee.core.aptget import EEAptGet +from ee.core.shellexec import EEShellExec +from ee.core.apt_repo import EERepo +import configparser +import os + + +class EEStackMigrateController(CementBaseController): + class Meta: + label = 'migrate' + stacked_on = 'stack' + stacked_type = 'nested' + description = ('Migrate stack safely') + arguments = [ + (['--mariadb'], + dict(help="Migrate database to MariaDB", + action='store_true')), + # (['--PHP'], + # dict(help="update to html site", action='store_true')), + ] + + @expose(hide=True) + def migrate_mariadb(self): + # Backup all database + EEMysql.backupAll(self) + + # Add MariaDB repo + Log.info(self, "Adding repository for MariaDB, please wait ...") + EERepo.add(self, repo_url=EEVariables.ee_mysql_repo) + Log.debug(self, 'Adding key for {0}' + .format(EEVariables.ee_mysql_repo)) + EERepo.add_key(self, '0xcbcb082a1bb943db', + keyserver="keyserver.ubuntu.com") + + config = configparser.ConfigParser() + config.read(os.path.expanduser("~")+'/.my.cnf') + try: + chars = config['client']['password'] + except Exception as e: + Log.error(self, "Error: process exited with error %s" + % e) + + Log.debug(self, "Pre-seeding MariaDB") + Log.debug(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + Log.debug(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + + # Install MariaDB + apt_packages = EEVariables.ee_mysql + Log.info(self, "Updating apt-cache, please wait ...") + EEAptGet.update(self) + Log.info(self, "Installing MariaDB, please wait ...") + EEAptGet.install(self, apt_packages) + + @expose(hide=True) + def default(self): + if ((not self.app.pargs.mariadb)): + self.app.args.print_help() + if self.app.pargs.mariadb: + if EEVariables.ee_mysql_host is not "localhost": + Log.error(self, "Remote MySQL found, EasyEngine will not " + "install MariaDB") + + if EEShellExec.cmd_exec(self, "mysqladmin ping") and (not + EEAptGet.is_installed(self, 'mariadb-server')): + + Log.info(self, "If your database size is big, " + "migration may take some time.") + Log.info(self, "During migration non nginx-cached parts of " + "your site may remain down") + start_migrate = input("Type \"mariadb\" to continue:") + if start_migrate != "mariadb": + Log.error(self, "Not starting migration") + self.migrate_mariadb() + else: + Log.error(self, "Your current MySQL is not alive or " + "you allready installed MariaDB") diff --git a/ee/core/mysql.py b/ee/core/mysql.py index daf10aec..3ae821fb 100644 --- a/ee/core/mysql.py +++ b/ee/core/mysql.py @@ -3,7 +3,9 @@ import pymysql import configparser from os.path import expanduser import sys +import os from ee.core.logging import Log +from ee.core.variables import EEVariables class EEMysql(): @@ -60,7 +62,42 @@ class EEMysql(): else: Log.error(self, '{0}'.format(errormsg)) + def backupAll(self): + import subprocess + try: + Log.info(self, "Backing up database at location: " + "/var/ee-mysqlbackup") + # Setup Nginx common directory + if not os.path.exists('/var/ee-mysqlbackup'): + Log.debug(self, 'Creating directory' + '/var/ee-mysqlbackup') + os.makedirs('/var/ee-mysqlbackup') -# def __del__(self): -# self.cur.close() -# self.conn.close() + db = subprocess.check_output(["mysql -Bse \'show databases\'"], + universal_newlines=True, + shell=True).split('\n') + for dbs in db: + if dbs == "": + continue + Log.info(self, "Backing up {0} database".format(dbs)) + p1 = subprocess.Popen("mysqldump {0}" + " --max_allowed_packet=1024M" + " --single-transaction".format(dbs), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + p2 = subprocess.Popen("gzip -c > /var/ee-mysqlbackup/{0}{1}.s" + "ql.gz".format(dbs, EEVariables.ee_date), + stdin=p1.stdout, + shell=True) + + # Allow p1 to receive a SIGPIPE if p2 exits + p1.stdout.close() + output = p1.stderr.read() + + if output == b'': + Log.debug(self, "done") + else: + Log.error(self, output) + except Exception as e: + Log.error(self, "Error: process exited with status %s" + % e) From cbd0c42f18583531cbab8838409701355fb3f38b Mon Sep 17 00:00:00 2001 From: gau1991 Date: Mon, 2 Mar 2015 19:16:14 +0530 Subject: [PATCH 2/7] Added autoremove after MariaDB setup --- ee/cli/plugins/stack_migrate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ee/cli/plugins/stack_migrate.py b/ee/cli/plugins/stack_migrate.py index 2e4becf4..2533aa76 100644 --- a/ee/cli/plugins/stack_migrate.py +++ b/ee/cli/plugins/stack_migrate.py @@ -73,6 +73,7 @@ class EEStackMigrateController(CementBaseController): EEAptGet.update(self) Log.info(self, "Installing MariaDB, please wait ...") EEAptGet.install(self, apt_packages) + EEAptGet.auto_remove(self) @expose(hide=True) def default(self): From 0ba07b5ef02a3c63d1916df4a518d0268461d39b Mon Sep 17 00:00:00 2001 From: harshadyeola Date: Mon, 2 Mar 2015 20:05:15 +0530 Subject: [PATCH 3/7] autocompletion for stack migrate --- config/bash_completion.d/ee_auto.rc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/config/bash_completion.d/ee_auto.rc b/config/bash_completion.d/ee_auto.rc index d7238ba7..d30a6446 100644 --- a/config/bash_completion.d/ee_auto.rc +++ b/config/bash_completion.d/ee_auto.rc @@ -35,7 +35,7 @@ _ee_complete() "stack") COMPREPLY=( $(compgen \ - -W "install purge reload remove restart start status stop" \ + -W "install purge reload remove restart start status stop migrate" \ -- $cur) ) ;; @@ -82,7 +82,11 @@ _ee_complete() -W "--nginx --php --mysql --postfix --memcache --dovecot" \ -- $cur) ) ;; - + "migrate") + COMPREPLY=( $(compgen \ + -W "--mariadb" \ + -- $cur) ) + ;; "list") COMPREPLY=( $(compgen \ -W "--enabled --disabled" \ From a87cdf7647a014b1bb9901cabeb590a7c9287a3e Mon Sep 17 00:00:00 2001 From: harshadyeola Date: Tue, 3 Mar 2015 13:54:45 +0530 Subject: [PATCH 4/7] minor fix --- config/bash_completion.d/ee_auto.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/bash_completion.d/ee_auto.rc b/config/bash_completion.d/ee_auto.rc index d30a6446..dcd33dc8 100644 --- a/config/bash_completion.d/ee_auto.rc +++ b/config/bash_completion.d/ee_auto.rc @@ -74,7 +74,7 @@ _ee_complete() # HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE "install" | "purge" | "remove" ) COMPREPLY=( $(compgen \ - -W "--web --admin --mail --nginx --php --mysql --postfix --wpcli --phpmyadmin --adminer --utils --memcache --dovecot" \ + -W "--web --admin --mail --nginx --php --mysql --postfix --wpcli --phpmyadmin --adminer --utils --memcache --dovecot --all" \ -- $cur) ) ;; "start" | "stop" | "reload" | "restart" | "status") From 7c4cb740cf7385a3922e73a35870923bd83623ae Mon Sep 17 00:00:00 2001 From: gau1991 Date: Tue, 3 Mar 2015 14:26:32 +0530 Subject: [PATCH 5/7] Rewritten aptget using sh module --- ee/core/aptget.py | 146 +++++++++------------------------------------- 1 file changed, 26 insertions(+), 120 deletions(-) diff --git a/ee/core/aptget.py b/ee/core/aptget.py index c9ea0b35..0ea763b8 100644 --- a/ee/core/aptget.py +++ b/ee/core/aptget.py @@ -14,24 +14,14 @@ class EEAptGet(): """ Similar to `apt-get upgrade` """ + global apt_get + apt_get = apt_get.bake("-y") try: - apt_cache = apt.cache.Cache() - import sys - orig_out = sys.stdout - sys.stdout = open(self.app.config.get('log.logging', 'file'), - encoding='utf-8', mode='a') - apt_cache.update(apt.progress.text.AcquireProgress()) - sys.stdout = orig_out - # success = (apt_cache.commit( - # apt.progress.text.AcquireProgress(), - # apt.progress.base.InstallProgress())) - # #apt_cache.close() - # return success - except AttributeError as e: - Log.error(self, 'AttributeError: ' + str(e)) - except Exception as e: - Log.debug(self, 'SystemError: ' + str(e)) - Log.error(self, 'Unable to Fetch update') + for line in apt_get.update(_iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') + except ErrorReturnCode as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to run apt-get update") def dist_upgrade(): """ @@ -54,112 +44,28 @@ class EEAptGet(): Log.error(self, 'Unable to Fetch update') def install(self, packages): - """ - Similar to `apt-get install` - """ - apt_pkg.init() - # #apt_pkg.PkgSystemLock() - global apt_cache - apt_cache = apt.cache.Cache() - - def install_package(self, package_name): - pkg = apt_cache[package_name.strip()] - if package_name.strip() in apt_cache: - if pkg.is_installed: - # apt_pkg.PkgSystemUnLock() - Log.debug(self, 'Trying to install a package that ' - 'is already installed (' + - package_name.strip() + ')') - # apt_cache.close() - return False - else: - try: - # print(pkg.name) - pkg.mark_install() - except Exception as e: - Log.debug(self, str(e)) - Log.error(self, str(e)) - else: - # apt_cache.close() - Log.error(self, 'Unknown package selected (' + - package_name.strip() + ')') - - for package in packages: - if not install_package(self, package): - continue - - if apt_cache.install_count > 0: - try: - # apt_pkg.PkgSystemUnLock() - orig_out = sys.stdout - sys.stdout = open(self.app.config.get('log.logging', 'file'), - encoding='utf-8', mode='a') - result = apt_cache.commit(apt.progress.text.AcquireProgress(), - apt.progress.base.InstallProgress()) - sys.stdout = orig_out - # apt_cache.close() - return result - except SystemError as e: - Log.debug(self, 'SystemError: ' + str(e)) - Log.error(self, 'SystemError: ' + str(e)) - # apt_cache.close() - except Exception as e: - Log.debug(self, str(e)) - Log.error(self, str(e)) + global apt_get + apt_get = apt_get.bake("-y") + try: + for line in apt_get.install(*packages, _iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') + except ErrorReturnCode as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to run apt-get install") def remove(self, packages, auto=False, purge=False): - """ - Similar to `apt-get remove/purge` - purge packages if purge=True - """ - apt_pkg.init() - # apt_pkg.PkgSystemLock() - global apt_cache - apt_cache = apt.cache.Cache() - - def remove_package(self, package_name, purge=False): - pkg = apt_cache[package_name.strip()] - if package_name.strip() in apt_cache: - if not pkg.is_installed: - # apt_pkg.PkgSystemUnLock() - Log.debug(self, 'Trying to uninstall a package ' - 'that is not installed (' + - package_name.strip() + ')') - return False - else: - try: - # print(pkg.name) - pkg.mark_delete(purge) - except SystemError as e: - Log.debug(self, 'SystemError: ' + str(e)) - return False + global apt_get + apt_get = apt_get.bake("-y") + try: + if purge == "True": + for line in apt_get.purge(*packages, _iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') else: - # apt_cache.close() - Log.error(self, 'Unknown package selected (' + - package_name.strip() + ')') - - for package in packages: - if not remove_package(self, package, purge=purge): - continue - - if apt_cache.delete_count > 0: - try: - # apt_pkg.PkgSystemUnLock() - orig_out = sys.stdout - sys.stdout = open(self.app.config.get('log.logging', 'file'), - encoding='utf-8', mode='a') - result = apt_cache.commit(apt.progress.text.AcquireProgress(), - apt.progress.base.InstallProgress()) - sys.stdout = orig_out - # apt_cache.close() - return result - except SystemError as e: - Log.debug(self, 'SystemError: ' + str(e)) - return False - except Exception as e: - Log.debug(self, str(e)) - Log.error(self, str(e)) - # apt_cache.close() + for line in apt_get.remove(*packages, _iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') + except ErrorReturnCode as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to run apt-get install") def auto_clean(self): """ From f0056f5b231f5a8c530493fe043f426866dcf562 Mon Sep 17 00:00:00 2001 From: gau1991 Date: Tue, 3 Mar 2015 14:49:54 +0530 Subject: [PATCH 6/7] Fixed Typo --- ee/core/aptget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/core/aptget.py b/ee/core/aptget.py index 0ea763b8..3c3eca65 100644 --- a/ee/core/aptget.py +++ b/ee/core/aptget.py @@ -65,7 +65,7 @@ class EEAptGet(): Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') except ErrorReturnCode as e: Log.debug(self, "{0}".format(e)) - Log.error(self, "Unable to run apt-get install") + Log.error(self, "Unable to remove packages") def auto_clean(self): """ From a9f6400110779da8b60333ca5fd1d8e5e76ba4d4 Mon Sep 17 00:00:00 2001 From: gau1991 Date: Tue, 3 Mar 2015 15:36:16 +0530 Subject: [PATCH 7/7] Updated Version to 3.0.5 --- ee/core/variables.py | 2 +- install | 2 +- setup.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ee/core/variables.py b/ee/core/variables.py index 53689112..5ab02cf7 100644 --- a/ee/core/variables.py +++ b/ee/core/variables.py @@ -12,7 +12,7 @@ class EEVariables(): """Intialization of core variables""" # EasyEngine version - ee_version = "3.0.4" + ee_version = "3.0.5" # EasyEngine packages versions ee_wp_cli = "0.18.0" diff --git a/install b/install index 55334045..11076335 100644 --- a/install +++ b/install @@ -5,7 +5,7 @@ # to update current EasyEngine from 2.x to 3.x old_ee_version="2.2.3" -new_ee_version="3.0.4" +new_ee_version="3.0.5" branch=$1 # Define echo function diff --git a/setup.py b/setup.py index 2153842f..6c22ce1f 100644 --- a/setup.py +++ b/setup.py @@ -54,15 +54,15 @@ except Exception as e: os.system("git config --global user.email {0}".format(ee_email)) setup(name='ee', - version='3.0.4', + version='3.0.5', description=long_description, long_description=long_description, classifiers=[], keywords='', author='rtCamp Soultions Pvt. LTD', - author_email='sys@rtcamp.com', + author_email='ee@rtcamp.com', url='http://rtcamp.com/easyengine', - license='GPL', + license='MIT', packages=find_packages(exclude=['ez_setup', 'examples', 'tests', 'templates']), include_package_data=True,