diff --git a/config/bash_completion.d/ee_auto.rc b/config/bash_completion.d/ee_auto.rc index d7238ba7..dcd33dc8 100644 --- a/config/bash_completion.d/ee_auto.rc +++ b/config/bash_completion.d/ee_auto.rc @@ -35,7 +35,7 @@ _ee_complete() "stack") COMPREPLY=( $(compgen \ - -W "install purge reload remove restart start status stop" \ + -W "install purge reload remove restart start status stop migrate" \ -- $cur) ) ;; @@ -74,7 +74,7 @@ _ee_complete() # HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE "install" | "purge" | "remove" ) COMPREPLY=( $(compgen \ - -W "--web --admin --mail --nginx --php --mysql --postfix --wpcli --phpmyadmin --adminer --utils --memcache --dovecot" \ + -W "--web --admin --mail --nginx --php --mysql --postfix --wpcli --phpmyadmin --adminer --utils --memcache --dovecot --all" \ -- $cur) ) ;; "start" | "stop" | "reload" | "restart" | "status") @@ -82,7 +82,11 @@ _ee_complete() -W "--nginx --php --mysql --postfix --memcache --dovecot" \ -- $cur) ) ;; - + "migrate") + COMPREPLY=( $(compgen \ + -W "--mariadb" \ + -- $cur) ) + ;; "list") COMPREPLY=( $(compgen \ -W "--enabled --disabled" \ diff --git a/ee/cli/plugins/stack.py b/ee/cli/plugins/stack.py index e0a24349..54a6fbb6 100644 --- a/ee/cli/plugins/stack.py +++ b/ee/cli/plugins/stack.py @@ -26,6 +26,7 @@ import pwd import grp import codecs from ee.cli.plugins.stack_services import EEStackStatusController +from ee.cli.plugins.stack_migrate import EEStackMigrateController from ee.core.logging import Log @@ -1523,6 +1524,7 @@ def load(app): # register the plugin class.. this only happens if the plugin is enabled handler.register(EEStackController) handler.register(EEStackStatusController) + handler.register(EEStackMigrateController) # register a hook (function) to run after arguments are parsed. hook.register('post_argument_parsing', ee_stack_hook) diff --git a/ee/cli/plugins/stack_migrate.py b/ee/cli/plugins/stack_migrate.py new file mode 100644 index 00000000..2533aa76 --- /dev/null +++ b/ee/cli/plugins/stack_migrate.py @@ -0,0 +1,100 @@ +from cement.core.controller import CementBaseController, expose +from cement.core import handler, hook +from ee.core.mysql import EEMysql +from ee.core.logging import Log +from ee.core.variables import EEVariables +from ee.core.aptget import EEAptGet +from ee.core.shellexec import EEShellExec +from ee.core.apt_repo import EERepo +import configparser +import os + + +class EEStackMigrateController(CementBaseController): + class Meta: + label = 'migrate' + stacked_on = 'stack' + stacked_type = 'nested' + description = ('Migrate stack safely') + arguments = [ + (['--mariadb'], + dict(help="Migrate database to MariaDB", + action='store_true')), + # (['--PHP'], + # dict(help="update to html site", action='store_true')), + ] + + @expose(hide=True) + def migrate_mariadb(self): + # Backup all database + EEMysql.backupAll(self) + + # Add MariaDB repo + Log.info(self, "Adding repository for MariaDB, please wait ...") + EERepo.add(self, repo_url=EEVariables.ee_mysql_repo) + Log.debug(self, 'Adding key for {0}' + .format(EEVariables.ee_mysql_repo)) + EERepo.add_key(self, '0xcbcb082a1bb943db', + keyserver="keyserver.ubuntu.com") + + config = configparser.ConfigParser() + config.read(os.path.expanduser("~")+'/.my.cnf') + try: + chars = config['client']['password'] + except Exception as e: + Log.error(self, "Error: process exited with error %s" + % e) + + Log.debug(self, "Pre-seeding MariaDB") + Log.debug(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password " + "password \" | " + "debconf-set-selections") + EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + Log.debug(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password_again " + "password \" | " + "debconf-set-selections") + EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 " + "mysql-server/root_password_again " + "password {chars}\" | " + "debconf-set-selections" + .format(chars=chars), + log=False) + + # Install MariaDB + apt_packages = EEVariables.ee_mysql + Log.info(self, "Updating apt-cache, please wait ...") + EEAptGet.update(self) + Log.info(self, "Installing MariaDB, please wait ...") + EEAptGet.install(self, apt_packages) + EEAptGet.auto_remove(self) + + @expose(hide=True) + def default(self): + if ((not self.app.pargs.mariadb)): + self.app.args.print_help() + if self.app.pargs.mariadb: + if EEVariables.ee_mysql_host is not "localhost": + Log.error(self, "Remote MySQL found, EasyEngine will not " + "install MariaDB") + + if EEShellExec.cmd_exec(self, "mysqladmin ping") and (not + EEAptGet.is_installed(self, 'mariadb-server')): + + Log.info(self, "If your database size is big, " + "migration may take some time.") + Log.info(self, "During migration non nginx-cached parts of " + "your site may remain down") + start_migrate = input("Type \"mariadb\" to continue:") + if start_migrate != "mariadb": + Log.error(self, "Not starting migration") + self.migrate_mariadb() + else: + Log.error(self, "Your current MySQL is not alive or " + "you allready installed MariaDB") diff --git a/ee/core/aptget.py b/ee/core/aptget.py index c9ea0b35..3c3eca65 100644 --- a/ee/core/aptget.py +++ b/ee/core/aptget.py @@ -14,24 +14,14 @@ class EEAptGet(): """ Similar to `apt-get upgrade` """ + global apt_get + apt_get = apt_get.bake("-y") try: - apt_cache = apt.cache.Cache() - import sys - orig_out = sys.stdout - sys.stdout = open(self.app.config.get('log.logging', 'file'), - encoding='utf-8', mode='a') - apt_cache.update(apt.progress.text.AcquireProgress()) - sys.stdout = orig_out - # success = (apt_cache.commit( - # apt.progress.text.AcquireProgress(), - # apt.progress.base.InstallProgress())) - # #apt_cache.close() - # return success - except AttributeError as e: - Log.error(self, 'AttributeError: ' + str(e)) - except Exception as e: - Log.debug(self, 'SystemError: ' + str(e)) - Log.error(self, 'Unable to Fetch update') + for line in apt_get.update(_iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') + except ErrorReturnCode as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to run apt-get update") def dist_upgrade(): """ @@ -54,112 +44,28 @@ class EEAptGet(): Log.error(self, 'Unable to Fetch update') def install(self, packages): - """ - Similar to `apt-get install` - """ - apt_pkg.init() - # #apt_pkg.PkgSystemLock() - global apt_cache - apt_cache = apt.cache.Cache() - - def install_package(self, package_name): - pkg = apt_cache[package_name.strip()] - if package_name.strip() in apt_cache: - if pkg.is_installed: - # apt_pkg.PkgSystemUnLock() - Log.debug(self, 'Trying to install a package that ' - 'is already installed (' + - package_name.strip() + ')') - # apt_cache.close() - return False - else: - try: - # print(pkg.name) - pkg.mark_install() - except Exception as e: - Log.debug(self, str(e)) - Log.error(self, str(e)) - else: - # apt_cache.close() - Log.error(self, 'Unknown package selected (' + - package_name.strip() + ')') - - for package in packages: - if not install_package(self, package): - continue - - if apt_cache.install_count > 0: - try: - # apt_pkg.PkgSystemUnLock() - orig_out = sys.stdout - sys.stdout = open(self.app.config.get('log.logging', 'file'), - encoding='utf-8', mode='a') - result = apt_cache.commit(apt.progress.text.AcquireProgress(), - apt.progress.base.InstallProgress()) - sys.stdout = orig_out - # apt_cache.close() - return result - except SystemError as e: - Log.debug(self, 'SystemError: ' + str(e)) - Log.error(self, 'SystemError: ' + str(e)) - # apt_cache.close() - except Exception as e: - Log.debug(self, str(e)) - Log.error(self, str(e)) + global apt_get + apt_get = apt_get.bake("-y") + try: + for line in apt_get.install(*packages, _iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') + except ErrorReturnCode as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to run apt-get install") def remove(self, packages, auto=False, purge=False): - """ - Similar to `apt-get remove/purge` - purge packages if purge=True - """ - apt_pkg.init() - # apt_pkg.PkgSystemLock() - global apt_cache - apt_cache = apt.cache.Cache() - - def remove_package(self, package_name, purge=False): - pkg = apt_cache[package_name.strip()] - if package_name.strip() in apt_cache: - if not pkg.is_installed: - # apt_pkg.PkgSystemUnLock() - Log.debug(self, 'Trying to uninstall a package ' - 'that is not installed (' + - package_name.strip() + ')') - return False - else: - try: - # print(pkg.name) - pkg.mark_delete(purge) - except SystemError as e: - Log.debug(self, 'SystemError: ' + str(e)) - return False + global apt_get + apt_get = apt_get.bake("-y") + try: + if purge == "True": + for line in apt_get.purge(*packages, _iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') else: - # apt_cache.close() - Log.error(self, 'Unknown package selected (' + - package_name.strip() + ')') - - for package in packages: - if not remove_package(self, package, purge=purge): - continue - - if apt_cache.delete_count > 0: - try: - # apt_pkg.PkgSystemUnLock() - orig_out = sys.stdout - sys.stdout = open(self.app.config.get('log.logging', 'file'), - encoding='utf-8', mode='a') - result = apt_cache.commit(apt.progress.text.AcquireProgress(), - apt.progress.base.InstallProgress()) - sys.stdout = orig_out - # apt_cache.close() - return result - except SystemError as e: - Log.debug(self, 'SystemError: ' + str(e)) - return False - except Exception as e: - Log.debug(self, str(e)) - Log.error(self, str(e)) - # apt_cache.close() + for line in apt_get.remove(*packages, _iter=True): + Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ') + except ErrorReturnCode as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to remove packages") def auto_clean(self): """ diff --git a/ee/core/mysql.py b/ee/core/mysql.py index daf10aec..3ae821fb 100644 --- a/ee/core/mysql.py +++ b/ee/core/mysql.py @@ -3,7 +3,9 @@ import pymysql import configparser from os.path import expanduser import sys +import os from ee.core.logging import Log +from ee.core.variables import EEVariables class EEMysql(): @@ -60,7 +62,42 @@ class EEMysql(): else: Log.error(self, '{0}'.format(errormsg)) + def backupAll(self): + import subprocess + try: + Log.info(self, "Backing up database at location: " + "/var/ee-mysqlbackup") + # Setup Nginx common directory + if not os.path.exists('/var/ee-mysqlbackup'): + Log.debug(self, 'Creating directory' + '/var/ee-mysqlbackup') + os.makedirs('/var/ee-mysqlbackup') -# def __del__(self): -# self.cur.close() -# self.conn.close() + db = subprocess.check_output(["mysql -Bse \'show databases\'"], + universal_newlines=True, + shell=True).split('\n') + for dbs in db: + if dbs == "": + continue + Log.info(self, "Backing up {0} database".format(dbs)) + p1 = subprocess.Popen("mysqldump {0}" + " --max_allowed_packet=1024M" + " --single-transaction".format(dbs), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + p2 = subprocess.Popen("gzip -c > /var/ee-mysqlbackup/{0}{1}.s" + "ql.gz".format(dbs, EEVariables.ee_date), + stdin=p1.stdout, + shell=True) + + # Allow p1 to receive a SIGPIPE if p2 exits + p1.stdout.close() + output = p1.stderr.read() + + if output == b'': + Log.debug(self, "done") + else: + Log.error(self, output) + except Exception as e: + Log.error(self, "Error: process exited with status %s" + % e) diff --git a/ee/core/variables.py b/ee/core/variables.py index 53689112..5ab02cf7 100644 --- a/ee/core/variables.py +++ b/ee/core/variables.py @@ -12,7 +12,7 @@ class EEVariables(): """Intialization of core variables""" # EasyEngine version - ee_version = "3.0.4" + ee_version = "3.0.5" # EasyEngine packages versions ee_wp_cli = "0.18.0" diff --git a/install b/install index e5d38980..40479df7 100644 --- a/install +++ b/install @@ -48,7 +48,7 @@ fi # Define variables for later use ee_branch=$1 readonly ee_version_old="2.2.3" -readonly ee_version_new="3.0.4" +readonly ee_version_new="3.0.5" readonly ee_log_dir=/var/log/ee/ readonly ee_install_log=/var/log/ee/install.log readonly ee_linux_distro=$(lsb_release -i | awk '{print $3}') @@ -89,9 +89,9 @@ function ee_install_dep() { ee_lib_echo "Installing required packages, please wait..." if [ "$ee_linux_distro" == "Ubuntu" ]; then - apt-get -y install gcc python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties software-properties-common || ee_lib_error "Unable to install pre depedencies, exit status " 1 + apt-get -y install gcc gzip python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties software-properties-common || ee_lib_error "Unable to install pre depedencies, exit status " 1 elif [ "$ee_linux_distro" == "Debian" ]; then - apt-get -y install gcc python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties || ee_lib_error "Unable to pre depedencies, exit status " 1 + apt-get -y install gcc gzip python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties || ee_lib_error "Unable to pre depedencies, exit status " 1 fi # Generating Locale diff --git a/setup.py b/setup.py index fc4b3538..ad938244 100644 --- a/setup.py +++ b/setup.py @@ -54,15 +54,15 @@ except Exception as e: os.system("git config --global user.email {0}".format(ee_email)) setup(name='ee', - version='3.0.4', + version='3.0.5', description=long_description, long_description=long_description, classifiers=[], keywords='', author='rtCamp Soultions Pvt. LTD', - author_email='sys@rtcamp.com', + author_email='ee@rtcamp.com', url='http://rtcamp.com/easyengine', - license='GPL', + license='MIT', packages=find_packages(exclude=['ez_setup', 'examples', 'tests', 'templates']), include_package_data=True,