Merge Python to Master

config/bash_completion.d/ee_auto.rc

@@ -35,7 +35,7 @@ _ee_complete()
 
             "stack")
                 COMPREPLY=( $(compgen \
-                              -W "install purge reload remove restart start status stop" \
+                              -W "install purge reload remove restart start status stop migrate" \
                               -- $cur) )
                 ;;
 
@@ -74,7 +74,7 @@ _ee_complete()
             # HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE
             "install" | "purge" | "remove" )
                 COMPREPLY=( $(compgen \
-                              -W "--web --admin --mail --nginx --php --mysql --postfix --wpcli --phpmyadmin --adminer --utils --memcache --dovecot" \
+                              -W "--web --admin --mail --nginx --php --mysql --postfix --wpcli --phpmyadmin --adminer --utils --memcache --dovecot --all" \
                               -- $cur) )
                 ;;
             "start" | "stop" | "reload" | "restart" | "status")
@@ -82,7 +82,11 @@ _ee_complete()
                               -W "--nginx --php --mysql --postfix --memcache --dovecot" \
                               -- $cur) )
                 ;;
-
+            "migrate")
+                COMPREPLY=( $(compgen \
+                              -W "--mariadb" \
+                              -- $cur) )
+                ;;
             "list")
                 COMPREPLY=( $(compgen \
                               -W "--enabled --disabled" \

ee/cli/plugins/stack.py

@@ -26,6 +26,7 @@ import pwd
 import grp
 import codecs
 from ee.cli.plugins.stack_services import EEStackStatusController
+from ee.cli.plugins.stack_migrate import EEStackMigrateController
 from ee.core.logging import Log
 
 
@@ -1523,6 +1524,7 @@ def load(app):
     # register the plugin class.. this only happens if the plugin is enabled
     handler.register(EEStackController)
     handler.register(EEStackStatusController)
+    handler.register(EEStackMigrateController)
 
     # register a hook (function) to run after arguments are parsed.
     hook.register('post_argument_parsing', ee_stack_hook)

ee/cli/plugins/stack_migrate.py

@@ -0,0 +1,100 @@
+from cement.core.controller import CementBaseController, expose
+from cement.core import handler, hook
+from ee.core.mysql import EEMysql
+from ee.core.logging import Log
+from ee.core.variables import EEVariables
+from ee.core.aptget import EEAptGet
+from ee.core.shellexec import EEShellExec
+from ee.core.apt_repo import EERepo
+import configparser
+import os
+
+
+class EEStackMigrateController(CementBaseController):
+    class Meta:
+        label = 'migrate'
+        stacked_on = 'stack'
+        stacked_type = 'nested'
+        description = ('Migrate stack safely')
+        arguments = [
+            (['--mariadb'],
+                dict(help="Migrate database to MariaDB",
+                     action='store_true')),
+            # (['--PHP'],
+            #    dict(help="update to html site", action='store_true')),
+            ]
+
+    @expose(hide=True)
+    def migrate_mariadb(self):
+        # Backup all database
+        EEMysql.backupAll(self)
+
+        # Add MariaDB repo
+        Log.info(self, "Adding repository for MariaDB, please wait ...")
+        EERepo.add(self, repo_url=EEVariables.ee_mysql_repo)
+        Log.debug(self, 'Adding key for {0}'
+                  .format(EEVariables.ee_mysql_repo))
+        EERepo.add_key(self, '0xcbcb082a1bb943db',
+                       keyserver="keyserver.ubuntu.com")
+
+        config = configparser.ConfigParser()
+        config.read(os.path.expanduser("~")+'/.my.cnf')
+        try:
+            chars = config['client']['password']
+        except Exception as e:
+            Log.error(self, "Error: process exited with error %s"
+                            % e)
+
+        Log.debug(self, "Pre-seeding MariaDB")
+        Log.debug(self, "echo \"mariadb-server-10.0 "
+                        "mysql-server/root_password "
+                        "password \" | "
+                        "debconf-set-selections")
+        EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 "
+                                   "mysql-server/root_password "
+                                   "password {chars}\" | "
+                                   "debconf-set-selections"
+                                   .format(chars=chars),
+                                   log=False)
+        Log.debug(self, "echo \"mariadb-server-10.0 "
+                        "mysql-server/root_password_again "
+                        "password \" | "
+                        "debconf-set-selections")
+        EEShellExec.cmd_exec(self, "echo \"mariadb-server-10.0 "
+                                   "mysql-server/root_password_again "
+                                   "password {chars}\" | "
+                                   "debconf-set-selections"
+                                   .format(chars=chars),
+                                   log=False)
+
+        # Install MariaDB
+        apt_packages = EEVariables.ee_mysql
+        Log.info(self, "Updating apt-cache, please wait ...")
+        EEAptGet.update(self)
+        Log.info(self, "Installing MariaDB, please wait ...")
+        EEAptGet.install(self, apt_packages)
+        EEAptGet.auto_remove(self)
+
+    @expose(hide=True)
+    def default(self):
+        if ((not self.app.pargs.mariadb)):
+            self.app.args.print_help()
+        if self.app.pargs.mariadb:
+            if EEVariables.ee_mysql_host is not "localhost":
+                Log.error(self, "Remote MySQL found, EasyEngine will not "
+                          "install MariaDB")
+
+            if EEShellExec.cmd_exec(self, "mysqladmin ping") and (not
+               EEAptGet.is_installed(self, 'mariadb-server')):
+
+                Log.info(self, "If your database size is big, "
+                         "migration may take some time.")
+                Log.info(self, "During migration non nginx-cached parts of "
+                         "your site may remain down")
+                start_migrate = input("Type \"mariadb\" to continue:")
+                if start_migrate != "mariadb":
+                    Log.error(self, "Not starting migration")
+                self.migrate_mariadb()
+            else:
+                Log.error(self, "Your current MySQL is not alive or "
+                          "you allready installed MariaDB")

ee/core/aptget.py

@@ -14,24 +14,14 @@ class EEAptGet():
         """
         Similar to `apt-get upgrade`
         """
+        global apt_get
+        apt_get = apt_get.bake("-y")
         try:
-            apt_cache = apt.cache.Cache()
+            for line in apt_get.update(_iter=True):
-            import sys
+                Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
-            orig_out = sys.stdout
+        except ErrorReturnCode as e:
-            sys.stdout = open(self.app.config.get('log.logging', 'file'),
+            Log.debug(self, "{0}".format(e))
-                              encoding='utf-8', mode='a')
+            Log.error(self, "Unable to run apt-get update")
-            apt_cache.update(apt.progress.text.AcquireProgress())
-            sys.stdout = orig_out
-            # success = (apt_cache.commit(
-            #            apt.progress.text.AcquireProgress(),
-            #            apt.progress.base.InstallProgress()))
-            # #apt_cache.close()
-            # return success
-        except AttributeError as e:
-            Log.error(self, 'AttributeError: ' + str(e))
-        except Exception as e:
-            Log.debug(self, 'SystemError:  ' + str(e))
-            Log.error(self, 'Unable to Fetch update')
 
     def dist_upgrade():
         """
@@ -54,112 +44,28 @@ class EEAptGet():
             Log.error(self, 'Unable to Fetch update')
 
     def install(self, packages):
-        """
+        global apt_get
-        Similar to `apt-get install`
+        apt_get = apt_get.bake("-y")
-        """
-        apt_pkg.init()
-        # #apt_pkg.PkgSystemLock()
-        global apt_cache
-        apt_cache = apt.cache.Cache()
-
-        def install_package(self, package_name):
-            pkg = apt_cache[package_name.strip()]
-            if package_name.strip() in apt_cache:
-                if pkg.is_installed:
-                    # apt_pkg.PkgSystemUnLock()
-                    Log.debug(self, 'Trying to install a package that '
-                              'is already installed (' +
-                              package_name.strip() + ')')
-                    # apt_cache.close()
-                    return False
-                else:
-                    try:
-                        # print(pkg.name)
-                        pkg.mark_install()
-                    except Exception as e:
-                        Log.debug(self, str(e))
-                        Log.error(self, str(e))
-            else:
-                # apt_cache.close()
-                Log.error(self, 'Unknown package selected (' +
-                          package_name.strip() + ')')
-
-        for package in packages:
-            if not install_package(self, package):
-                continue
-
-        if apt_cache.install_count > 0:
         try:
-                # apt_pkg.PkgSystemUnLock()
+            for line in apt_get.install(*packages, _iter=True):
-                orig_out = sys.stdout
+                Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
-                sys.stdout = open(self.app.config.get('log.logging', 'file'),
+        except ErrorReturnCode as e:
-                                  encoding='utf-8', mode='a')
+            Log.debug(self, "{0}".format(e))
-                result = apt_cache.commit(apt.progress.text.AcquireProgress(),
+            Log.error(self, "Unable to run apt-get install")
-                                          apt.progress.base.InstallProgress())
-                sys.stdout = orig_out
-                # apt_cache.close()
-                return result
-            except SystemError as e:
-                Log.debug(self, 'SystemError: ' + str(e))
-                Log.error(self, 'SystemError: ' + str(e))
-                # apt_cache.close()
-            except Exception as e:
-                Log.debug(self, str(e))
-                Log.error(self, str(e))
 
     def remove(self, packages, auto=False, purge=False):
-        """
+        global apt_get
-            Similar to `apt-get remove/purge`
+        apt_get = apt_get.bake("-y")
-            purge packages if purge=True
-        """
-        apt_pkg.init()
-        # apt_pkg.PkgSystemLock()
-        global apt_cache
-        apt_cache = apt.cache.Cache()
-
-        def remove_package(self, package_name, purge=False):
-            pkg = apt_cache[package_name.strip()]
-            if package_name.strip() in apt_cache:
-                if not pkg.is_installed:
-                    # apt_pkg.PkgSystemUnLock()
-                    Log.debug(self, 'Trying to uninstall a package '
-                              'that is not installed (' +
-                              package_name.strip() + ')')
-                    return False
-                else:
         try:
-                        # print(pkg.name)
+            if purge == "True":
-                        pkg.mark_delete(purge)
+                for line in apt_get.purge(*packages, _iter=True):
-                    except SystemError as e:
+                    Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
-                        Log.debug(self, 'SystemError: ' + str(e))
-                        return False
             else:
-                # apt_cache.close()
+                for line in apt_get.remove(*packages, _iter=True):
-                Log.error(self, 'Unknown package selected (' +
+                    Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
-                          package_name.strip() + ')')
+        except ErrorReturnCode as e:
-
+            Log.debug(self, "{0}".format(e))
-        for package in packages:
+            Log.error(self, "Unable to remove packages")
-            if not remove_package(self, package, purge=purge):
-                continue
-
-        if apt_cache.delete_count > 0:
-            try:
-                # apt_pkg.PkgSystemUnLock()
-                orig_out = sys.stdout
-                sys.stdout = open(self.app.config.get('log.logging', 'file'),
-                                  encoding='utf-8', mode='a')
-                result = apt_cache.commit(apt.progress.text.AcquireProgress(),
-                                          apt.progress.base.InstallProgress())
-                sys.stdout = orig_out
-                # apt_cache.close()
-                return result
-            except SystemError as e:
-                Log.debug(self, 'SystemError: ' + str(e))
-                return False
-            except Exception as e:
-                Log.debug(self, str(e))
-                Log.error(self, str(e))
-                # apt_cache.close()
 
     def auto_clean(self):
         """

ee/core/mysql.py

@@ -3,7 +3,9 @@ import pymysql
 import configparser
 from os.path import expanduser
 import sys
+import os
 from ee.core.logging import Log
+from ee.core.variables import EEVariables
 
 
 class EEMysql():
@@ -60,7 +62,42 @@ class EEMysql():
                 else:
                     Log.error(self, '{0}'.format(errormsg))
 
+    def backupAll(self):
+        import subprocess
+        try:
+            Log.info(self, "Backing up database at location: "
+                     "/var/ee-mysqlbackup")
+            # Setup Nginx common directory
+            if not os.path.exists('/var/ee-mysqlbackup'):
+                Log.debug(self, 'Creating directory'
+                          '/var/ee-mysqlbackup')
+                os.makedirs('/var/ee-mysqlbackup')
+
+            db = subprocess.check_output(["mysql -Bse \'show databases\'"],
+                                         universal_newlines=True,
+                                         shell=True).split('\n')
+            for dbs in db:
+                if dbs == "":
+                    continue
+                Log.info(self, "Backing up {0} database".format(dbs))
+                p1 = subprocess.Popen("mysqldump {0}"
+                                      " --max_allowed_packet=1024M"
+                                      " --single-transaction".format(dbs),
+                                      stdout=subprocess.PIPE,
+                                      stderr=subprocess.PIPE, shell=True)
+                p2 = subprocess.Popen("gzip -c > /var/ee-mysqlbackup/{0}{1}.s"
+                                      "ql.gz".format(dbs, EEVariables.ee_date),
+                                      stdin=p1.stdout,
+                                      shell=True)
+
+                # Allow p1 to receive a SIGPIPE if p2 exits
+                p1.stdout.close()
+                output = p1.stderr.read()
 
-#    def __del__(self):
+                if output == b'':
-#        self.cur.close()
+                    Log.debug(self, "done")
-#        self.conn.close()
+                else:
+                    Log.error(self, output)
+        except Exception as e:
+            Log.error(self, "Error: process exited with status %s"
+                            % e)

ee/core/variables.py

@@ -12,7 +12,7 @@ class EEVariables():
     """Intialization of core variables"""
 
     # EasyEngine version
-    ee_version = "3.0.4"
+    ee_version = "3.0.5"
 
     # EasyEngine packages versions
     ee_wp_cli = "0.18.0"

install

@@ -48,7 +48,7 @@ fi
 # Define variables for later use
 ee_branch=$1
 readonly ee_version_old="2.2.3"
-readonly ee_version_new="3.0.4"
+readonly ee_version_new="3.0.5"
 readonly ee_log_dir=/var/log/ee/
 readonly ee_install_log=/var/log/ee/install.log
 readonly ee_linux_distro=$(lsb_release -i | awk '{print $3}')
@@ -89,9 +89,9 @@ function ee_install_dep()
 {
     ee_lib_echo "Installing required packages, please wait..."
     if [ "$ee_linux_distro" == "Ubuntu" ]; then
-        apt-get -y install gcc python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties software-properties-common || ee_lib_error "Unable to install pre depedencies, exit status " 1
+        apt-get -y install gcc gzip python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties software-properties-common || ee_lib_error "Unable to install pre depedencies, exit status " 1
     elif [ "$ee_linux_distro" == "Debian" ]; then
-        apt-get -y install gcc python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties || ee_lib_error "Unable to pre depedencies, exit status " 1
+        apt-get -y install gcc gzip python3 python3-apt python3-setuptools python3-dev sqlite3 git tar python-software-properties || ee_lib_error "Unable to pre depedencies, exit status " 1
     fi
 
     # Generating Locale

setup.py

@@ -54,15 +54,15 @@ except Exception as e:
     os.system("git config --global user.email {0}".format(ee_email))
 
 setup(name='ee',
-      version='3.0.4',
+      version='3.0.5',
       description=long_description,
       long_description=long_description,
       classifiers=[],
       keywords='',
       author='rtCamp Soultions Pvt. LTD',
-      author_email='sys@rtcamp.com',
+      author_email='ee@rtcamp.com',
       url='http://rtcamp.com/easyengine',
-      license='GPL',
+      license='MIT',
       packages=find_packages(exclude=['ez_setup', 'examples', 'tests',
                                       'templates']),
       include_package_data=True,