Browse Source

Merge branch 'python'

bugfixes
gau1991 10 years ago
parent
commit
fff68d1183
  1. 6
      .travis.yml
  2. 28
      config/bash_completion.d/ee_auto.rc
  3. 20
      ee/cli/ext/ee_outputhandler.py
  4. 25
      ee/cli/main.py
  5. 123
      ee/cli/plugins/debug.py
  6. 9
      ee/cli/plugins/stack.py
  7. 18
      ee/cli/plugins/stack_migrate.py
  8. 79
      ee/core/aptget.py

6
.travis.yml

@ -64,10 +64,10 @@ script:
- sudo ee site create site12.in --wpfc --wpsubdomain
- sudo ee site create site12.net --wpfc --wpsubdomain
- sudo ee debug
- sudo ee debug --stop
- sudo ee debug --all
- sudo ee debug --all=off
- sudo ee debug site12.net
- sudo ee debug site12.net --stop
- sudo ee debug site12.net --all=off
- sudo ee site create 1.com --html
- sudo ee site create 2.com --php
- sudo ee site create 3.com --mysql

28
config/bash_completion.d/ee_auto.rc

@ -29,7 +29,7 @@ _ee_complete()
# IF YOU HAD ANOTHER CONTROLLER, YOU'D HANDLE THAT HERE
"debug")
COMPREPLY=( $(compgen \
-W "$(command find /etc/nginx/sites-enabled/ -type l -printf "%P " 2> /dev/null) --start --nginx --php --fpm --mysql -i --interactive --stop " \
-W "$(command find /etc/nginx/sites-enabled/ -type l -printf "%P " 2> /dev/null) --nginx --php --fpm --mysql -i --interactive --all --nginx=off --php=off --fpm=off --mysql=off --all=off " \
-- $cur) )
;;
@ -59,7 +59,7 @@ _ee_complete()
"log")
COMPREPLY=( $(compgen \
-W "--mysql --php --nginx --all" \
-W "$(command find /etc/nginx/sites-enabled/ -type l -printf "%P " 2> /dev/null) --mysql --php --nginx --all" \
-- $cur) )
;;
@ -110,13 +110,22 @@ _ee_complete()
esac
if [ ${COMP_WORDS[1]} == "debug" ] && ([ "$prev" != "--start" ] || [ "$prev" != "--nginx" ] || [ "$prev" != "--php" ] || [ "$prev" != "--fpm" ] || [ "$prev" != "--mysql" ] || [ "$prev" != "-i" ] || ["$prev" != "--interactive" ] || ["$prev" != "--stop" ]); then
retlist="--start --stop --wp --rewrite -i"
retlist="--all --wp --rewrite -i --all=off --wp=off --rewrite=off"
ret="${retlist[@]/$prev}"
COMPREPLY=( $(compgen \
-W "$(echo $ret)" \
-- $cur) )
fi
if [ ${COMP_WORDS[1]} == "log" ] && ([ "$prev" != "--access" ] || [ "$prev" != "--nginx" ] || [ "$prev" != "--php" ] || [ "$prev" != "--fpm" ] || [ "$prev" != "--mysql" ] || [ "$prev" != "-i" ] || ["$prev" != "--interactive" ] || ["$prev" != "--stop" ]); then
retlist="--all --wp --rewrite -i --all=off --wp=off --rewrite=off"
ret="${retlist[@]/$prev}"
COMPREPLY=( $(compgen \
-W "$(echo $ret)" \
-- $cur) )
fi
elif [ $COMP_CWORD -eq 4 ]; then
case "$mprev" in
# HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE
@ -145,9 +154,16 @@ _ee_complete()
case "$prev" in
"--wp" | "--wpsubdir" | "--wpsubdomain")
COMPREPLY=( $(compgen \
-W "--w3tc --wpfc --wpsc" \
-- $cur) )
if [ ${COMP_WORDS[1]} != "debug" ]; then
retlist="--w3tc --wpfc --wpsc"
else
retlist="--wp=off --rewrite --rewrite=off -i --interactive"
fi
ret="${retlist[@]/$prev}"
COMPREPLY=( $(compgen \
-W "$(echo $ret)" \
-- $cur) )
;;
"--web" | "--admin" | "--mail" | "--nginx" | "--php" | "--mysql" | "--postfix" | "--wpcli" | "--phpmyadmin" | "--adminer" | "--utils" | "--memcache" | "--dovecot")

20
ee/cli/ext/ee_outputhandler.py

@ -0,0 +1,20 @@
# Based on https://github.com/datafolklabs/cement/issues/295
# To avoid encoding releated error,we defined our custom output handler
# I hope we will remove this when we upgarde to Cement 2.6 (Not released yet)
import os
from cement.utils import fs
from cement.ext.ext_mustache import MustacheOutputHandler
class EEOutputHandler(MustacheOutputHandler):
class Meta:
label = 'ee_output_handler'
def _load_template_from_file(self, path):
for templ_dir in self.app._meta.template_dirs:
full_path = fs.abspath(os.path.join(templ_dir, path))
if os.path.exists(full_path):
self.app.log.debug('loading template file %s' % full_path)
return open(full_path, encoding='utf-8', mode='r').read()
else:
continue

25
ee/cli/main.py

@ -13,10 +13,9 @@ else:
from cement.core import foundation
from cement.utils.misc import init_defaults
from cement.core.exc import FrameworkError, CaughtSignal
from cement.utils import fs
from cement.ext.ext_mustache import MustacheOutputHandler
from cement.ext.ext_argparse import ArgParseArgumentHandler
from ee.core import exc
from ee.cli.ext.ee_outputhandler import EEOutputHandler
# Application default. Should update config/ee.conf to reflect any
# changes, or additions here.
@ -31,22 +30,12 @@ defaults['ee']['plugin_dir'] = '/var/lib/ee/plugins'
# External templates (generally, do not ship with application code)
defaults['ee']['template_dir'] = '/var/lib/ee/templates'
# Based on https://github.com/datafolklabs/cement/issues/295
# To avoid encoding releated error,we defined our custom output handler
# I hope we will remove this when we upgarde to Cement 2.6 (Not released yet)
class EEOutputHandler(MustacheOutputHandler):
class EEArgHandler(ArgParseArgumentHandler):
class Meta:
label = 'ee_output_handler'
label = 'ee_args_handler'
def _load_template_from_file(self, path):
for templ_dir in self.app._meta.template_dirs:
full_path = fs.abspath(os.path.join(templ_dir, path))
if os.path.exists(full_path):
self.app.log.debug('loading template file %s' % full_path)
return open(full_path, encoding='utf-8', mode='r').read()
else:
continue
def error(self, message):
super(EEArgHandler, self).error("unknown args")
class EEApp(foundation.CementApp):
@ -72,6 +61,8 @@ class EEApp(foundation.CementApp):
# default output handler
output_handler = EEOutputHandler
arg_handler = EEArgHandler
debug = TOGGLE_DEBUG

123
ee/cli/plugins/debug.py

@ -14,6 +14,27 @@ import glob
import signal
import subprocess
usage = """
Usage: ee debug {<sitename>} {arguments}
arguments usage:
--all --all=on start debugging all server parameters.
--all=off stop debugging all server parameters
--nginx --nginx=on start debugging nginx server configuration for site
--nginx=off stop debugging nginx server configuration for site
--rewrite --rewrite=on start debugging nginx rewrite rules for site
--rewrite=off stop debugging nginx rewrite rules for site
--php --php=on start debugging server php configuration
--php=off stop debugging server php configuration
--fpm --fpm=on start debugging fastcgi configuration
--fpm=off stop debugging fastcgi configuration
--mysql --mysql=on start debugging mysql server
--mysql=off stop debugging mysql server
--wp --wp=on start wordpress debugging
--wp=off stop wordpress debugging
"""
def debug_plugin_hook(app):
# do something with the ``app`` object here.
@ -32,17 +53,34 @@ class EEDebugController(CementBaseController):
(['--start'],
dict(help='Start debug', action='store_true')),
(['--nginx'],
dict(help='Debug Nginx', action='store_true')),
dict(help='start/stop debugging nginx server '
'configuration for site',
action='store' or 'store_const',
choices=('on', 'off'), const='on', nargs='?')),
(['--php'],
dict(help='Debug PHP', action='store_true')),
dict(help='start/stop debugging server php configuration',
action='store' or 'store_const',
choices=('on', 'off'), const='on', nargs='?')),
(['--fpm'],
dict(help='Debug FastCGI', action='store_true')),
dict(help='start/stop debugging fastcgi configuration',
action='store' or 'store_const',
choices=('on', 'off'), const='on', nargs='?')),
(['--mysql'],
dict(help='Debug MySQL', action='store_true')),
dict(help='start/stop debugging mysql server',
action='store' or 'store_const',
choices=('on', 'off'), const='on', nargs='?')),
(['--wp'],
dict(help='Debug WordPress sites', action='store_true')),
dict(help='start/stop wordpress debugging for site',
action='store' or 'store_const', choices=('on', 'off'),
const='on', nargs='?')),
(['--rewrite'],
dict(help='Debug Nginx rewrite rules', action='store_true')),
dict(help='start/stop debugging nginx rewrite rules for site',
action='store' or 'store_const', choices=('on', 'off'),
const='on', nargs='?')),
(['--all'],
dict(help='start/stop debugging all server parameters',
action='store' or 'store_const', choices=('on', 'off'),
const='on', nargs='?')),
(['-i', '--interactive'],
dict(help='Interactive debug', action='store_true')),
(['--import-slow-log-interval'],
@ -56,7 +94,7 @@ class EEDebugController(CementBaseController):
def debug_nginx(self):
"""Start/Stop Nginx debug"""
# start global debug
if self.start and not self.app.pargs.site_name:
if (self.app.pargs.nginx == 'on' and not self.app.pargs.site_name):
try:
debug_address = (self.app.config.get('stack', 'ip-address')
.split())
@ -79,7 +117,7 @@ class EEDebugController(CementBaseController):
self.msg = self.msg + ["/var/log/nginx/*.error.log"]
# stop global debug
elif not self.start and not self.app.pargs.site_name:
elif (self.app.pargs.nginx == 'off' and not self.app.pargs.site_name):
if "debug_connection " in open('/etc/nginx/nginx.conf',
encoding='utf-8').read():
Log.info(self, "Disabling Nginx debug connections")
@ -90,7 +128,7 @@ class EEDebugController(CementBaseController):
Log.info(self, "Nginx debug connection already disabled")
# start site specific debug
elif self.start and self.app.pargs.site_name:
elif (self.app.pargs.nginx == 'on'and self.app.pargs.site_name):
config_path = ("/etc/nginx/sites-available/{0}"
.format(self.app.pargs.site_name))
if os.path.isfile(config_path):
@ -115,7 +153,7 @@ class EEDebugController(CementBaseController):
.format(self.app.pargs.site_name))
# stop site specific debug
elif not self.start and self.app.pargs.site_name:
elif (self.app.pargs.nginx == 'off' and self.app.pargs.site_name):
config_path = ("/etc/nginx/sites-available/{0}"
.format(self.app.pargs.site_name))
if os.path.isfile(config_path):
@ -139,7 +177,7 @@ class EEDebugController(CementBaseController):
def debug_php(self):
"""Start/Stop PHP debug"""
# PHP global debug start
if self.start:
if (self.app.pargs.php == 'on' and not self.app.pargs.site_name):
if not (EEShellExec.cmd_exec(self, "sed -n \"/upstream php"
"{/,/}/p \" /etc/nginx/"
"conf.d/upstream.conf "
@ -160,7 +198,7 @@ class EEDebugController(CementBaseController):
self.msg = self.msg + ['/var/log/php5/slow.log']
# PHP global debug stop
else:
elif (self.app.pargs.php == 'off' and not self.app.pargs.site_name):
if EEShellExec.cmd_exec(self, " sed -n \"/upstream php {/,/}/p\" "
"/etc/nginx/conf.d/upstream.conf "
"| grep 9001"):
@ -181,7 +219,7 @@ class EEDebugController(CementBaseController):
def debug_fpm(self):
"""Start/Stop PHP5-FPM debug"""
# PHP5-FPM start global debug
if self.start:
if (self.app.pargs.php == 'on'and not self.app.pargs.site_name):
if not EEShellExec.cmd_exec(self, "grep \"log_level = debug\" "
"/etc/php5/fpm/php-fpm.conf"):
Log.info(self, "Setting up PHP5-FPM log_level = debug")
@ -202,7 +240,7 @@ class EEDebugController(CementBaseController):
self.msg = self.msg + ['/var/log/php5/fpm.log']
# PHP5-FPM stop global debug
else:
elif (self.app.pargs.php == 'on' and not self.app.pargs.site_name):
if EEShellExec.cmd_exec(self, "grep \"log_level = debug\" "
"/etc/php5/fpm/php-fpm.conf"):
Log.info(self, "Disabling PHP5-FPM log_level = debug")
@ -225,7 +263,7 @@ class EEDebugController(CementBaseController):
def debug_mysql(self):
"""Start/Stop MySQL debug"""
# MySQL start global debug
if self.start:
if (self.app.pargs.mysql == 'on' and not self.app.pargs.site_name):
if not EEShellExec.cmd_exec(self, "mysql -e \"show variables like"
" \'slow_query_log\';\" | "
"grep ON"):
@ -258,7 +296,7 @@ class EEDebugController(CementBaseController):
self.msg = self.msg + ['/var/log/mysql/mysql-slow.log']
# MySQL stop global debug
else:
elif (self.app.pargs.mysql == 'off' and not self.app.pargs.site_name):
if EEShellExec.cmd_exec(self, "mysql -e \"show variables like \'"
"slow_query_log\';\" | grep ON"):
Log.info(self, "Disabling MySQL slow log")
@ -276,7 +314,7 @@ class EEDebugController(CementBaseController):
@expose(hide=True)
def debug_wp(self):
"""Start/Stop WordPress debug"""
if self.start and self.app.pargs.site_name:
if (self.app.pargs.wp == 'on' and self.app.pargs.site_name):
wp_config = ("{0}/{1}/wp-config.php"
.format(EEVariables.ee_webroot,
self.app.pargs.site_name))
@ -316,7 +354,7 @@ class EEDebugController(CementBaseController):
Log.info(self, "{0} domain not valid"
.format(self.app.pargs.site_name))
elif not self.start and self.app.pargs.site_name:
elif (self.app.pargs.wp == 'off' and self.app.pargs.site_name):
wp_config = ("{0}{1}/wp-config.php"
.format(EEVariables.ee_webroot,
self.app.pargs.site_name))
@ -347,7 +385,7 @@ class EEDebugController(CementBaseController):
def debug_rewrite(self):
"""Start/Stop Nginx rewrite rules debug"""
# Start Nginx rewrite debug globally
if self.start and not self.app.pargs.site_name:
if (self.app.pargs.rewrite == 'on' and not self.app.pargs.site_name):
if not EEShellExec.cmd_exec(self, "grep \"rewrite_log on;\" "
"/etc/nginx/nginx.conf"):
Log.info(self, "Setting up Nginx rewrite logs")
@ -361,7 +399,8 @@ class EEDebugController(CementBaseController):
self.msg = self.msg + ['/var/log/nginx/*.error.log']
# Stop Nginx rewrite debug globally
elif not self.start and not self.app.pargs.site_name:
elif (self.app.pargs.rewrite == 'off'
and not self.app.pargs.site_name):
if EEShellExec.cmd_exec(self, "grep \"rewrite_log on;\" "
"/etc/nginx/nginx.conf"):
Log.info(self, "Disabling Nginx rewrite logs")
@ -371,7 +410,7 @@ class EEDebugController(CementBaseController):
else:
Log.info(self, "Nginx rewrite logs already disabled")
# Start Nginx rewrite for site
elif self.start and self.app.pargs.site_name:
elif (self.app.pargs.rewrite == 'on' and self.app.pargs.site_name):
config_path = ("/etc/nginx/sites-available/{0}"
.format(self.app.pargs.site_name))
if not EEShellExec.cmd_exec(self, "grep \"rewrite_log on;\" {0}"
@ -394,7 +433,7 @@ class EEDebugController(CementBaseController):
self.app.pargs.site_name)]
# Stop Nginx rewrite for site
elif not self.start and self.app.pargs.site_name:
elif (self.app.pargs.rewrite == 'off' and self.app.pargs.site_name):
config_path = ("/etc/nginx/sites-available/{0}"
.format(self.app.pargs.site_name))
if EEShellExec.cmd_exec(self, "grep \"rewrite_log on;\" {0}"
@ -442,32 +481,48 @@ class EEDebugController(CementBaseController):
@expose(hide=True)
def default(self):
"""Default function of debug"""
self.start = True
# self.start = True
self.interactive = False
self.msg = []
self.trigger_nginx = False
self.trigger_php = False
if self.app.pargs.stop:
self.start = False
if ((not self.app.pargs.nginx) and (not self.app.pargs.php)
and (not self.app.pargs.fpm) and (not self.app.pargs.mysql)
and (not self.app.pargs.wp) and (not self.app.pargs.rewrite)
and (not self.app.pargs.all)
and (not self.app.pargs.site_name)):
self.app.pargs.nginx = True
self.app.pargs.php = True
self.app.pargs.fpm = True
self.app.pargs.mysql = True
self.app.pargs.rewrite = True
if self.app.pargs.stop or self.app.pargs.start:
print("--start/stop option is deprecated in ee3.0.5", usage)
else:
print(usage)
if self.app.pargs.all == 'on':
if self.app.pargs.site_name:
self.app.pargs.wp = 'on'
self.app.pargs.nginx = 'on'
self.app.pargs.php = 'on'
self.app.pargs.fpm = 'on'
self.app.pargs.mysql = 'on'
self.app.pargs.rewrite = 'on'
if self.app.pargs.all == 'off':
if self.app.pargs.site_name:
self.app.pargs.wp = 'off'
self.app.pargs.nginx = 'off'
self.app.pargs.php = 'off'
self.app.pargs.fpm = 'off'
self.app.pargs.mysql = 'off'
self.app.pargs.rewrite = 'off'
if ((not self.app.pargs.nginx) and (not self.app.pargs.php)
and (not self.app.pargs.fpm) and (not self.app.pargs.mysql)
and (not self.app.pargs.wp) and (not self.app.pargs.rewrite)
and self.app.pargs.site_name):
self.app.pargs.nginx = True
self.app.pargs.wp = True
self.app.pargs.rewrite = True
print(usage)
# self.app.pargs.nginx = 'on'
# self.app.pargs.wp = 'on'
# self.app.pargs.rewrite = 'on'
if self.app.pargs.nginx:
self.debug_nginx()

9
ee/cli/plugins/stack.py

@ -546,6 +546,15 @@ class EEStackController(CementBaseController):
"/dovecot.pem")
# Custom Dovecot configuration by EasyEngine
data = dict()
Log.debug(self, "Writting configuration into file"
"/etc/dovecot/conf.d/auth-sql.conf.ext ")
ee_dovecot = open('/etc/dovecot/conf.d/auth-sql.conf.ext',
encoding='utf-8', mode='w')
self.app.render((data), 'auth-sql-conf.mustache',
out=ee_dovecot)
ee_dovecot.close()
data = dict(email=EEVariables.ee_email)
Log.debug(self, "Writting configuration into file"
"/etc/dovecot/conf.d/99-ee.conf ")

18
ee/cli/plugins/stack_migrate.py

@ -6,6 +6,7 @@ from ee.core.variables import EEVariables
from ee.core.aptget import EEAptGet
from ee.core.shellexec import EEShellExec
from ee.core.apt_repo import EERepo
from ee.core.services import EEService
import configparser
import os
@ -75,7 +76,17 @@ class EEStackMigrateController(CementBaseController):
log=False)
# Install MariaDB
apt_packages = EEVariables.ee_mysql + ["php5-mysql"]
apt_packages = EEVariables.ee_mysql
# If PHP is installed then install php5-mysql
if EEAptGet.is_installed(self, "php5-fpm"):
apt_packages = apt_packages + ["php5-mysql"]
# If mail server is installed then install dovecot-sql and postfix-sql
if EEAptGet.is_installed(self, "dovecot-core"):
apt_packages = apt_packages + ["dovecot-mysql", "postfix-mysql",
"libclass-dbi-mysql-perl"]
Log.info(self, "Updating apt-cache, please wait ...")
EEAptGet.update(self)
Log.info(self, "Installing MariaDB, please wait ...")
@ -83,6 +94,11 @@ class EEStackMigrateController(CementBaseController):
EEAptGet.auto_remove(self)
EEAptGet.install(self, apt_packages)
# Restart dovecot and postfix if installed
if EEAptGet.is_installed(self, "dovecot-core"):
EEService.restart_service(self, 'dovecot')
EEService.restart_service(self, 'postfix')
@expose(hide=True)
def default(self):
if ((not self.app.pargs.mariadb)):

79
ee/core/aptget.py

@ -2,6 +2,7 @@
import apt
import apt_pkg
import sys
import subprocess
from ee.core.logging import Log
from sh import apt_get
from sh import ErrorReturnCode
@ -14,14 +15,22 @@ class EEAptGet():
"""
Similar to `apt-get upgrade`
"""
global apt_get
apt_get = apt_get.bake("-y")
try:
for line in apt_get.update(_iter=True):
Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to run apt-get update")
with open('/var/log/ee/ee.log', 'a') as f:
proc = subprocess.Popen('apt-get update',
shell=True,
stdin=None, stdout=f, stderr=f,
executable="/bin/bash")
proc.wait()
if proc.returncode == 0:
return True
else:
Log.error(self, "Unable to run apt-get update")
except Exception as e:
Log.error(self, "Error while installing packages, "
"apt-get exited with error")
def dist_upgrade():
"""
@ -44,30 +53,48 @@ class EEAptGet():
Log.error(self, 'Unable to Fetch update')
def install(self, packages):
global apt_get
apt_get = apt_get.bake("-y")
all_packages = ' '.join(packages)
try:
for line in apt_get.install("-o",
"Dpkg::Options::=--force-confold",
*packages, _iter=True):
Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to run apt-get install")
with open('/var/log/ee/ee.log', 'a') as f:
proc = subprocess.Popen("apt-get install -o Dpkg::Options::=--"
"force-confold -y {0}"
.format(all_packages), shell=True,
stdin=None, stdout=f, stderr=f,
executable="/bin/bash")
proc.wait()
if proc.returncode == 0:
return True
else:
Log.error(self, "Unable to run apt-get install")
except Exception as e:
Log.error(self, "Error while installing packages, "
"apt-get exited with error")
def remove(self, packages, auto=False, purge=False):
global apt_get
apt_get = apt_get.bake("-y")
all_packages = ' '.join(packages)
try:
if purge:
for line in apt_get.purge(*packages, _iter=True):
Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
with open('/var/log/ee/ee.log', 'a') as f:
if purge:
proc = subprocess.Popen('apt-get purge -y {0}'
.format(all_packages), shell=True,
stdin=None, stdout=f, stderr=f,
executable="/bin/bash")
else:
proc = subprocess.Popen('apt-get remove -y {0}'
.format(all_packages), shell=True,
stdin=None, stdout=f, stderr=f,
executable="/bin/bash")
proc.wait()
if proc.returncode == 0:
return True
else:
for line in apt_get.remove(*packages, _iter=True):
Log.info(self, Log.ENDC+line+Log.OKBLUE, end=' ')
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to remove packages")
Log.error(self, "Unable to run apt-get remove/purge")
except Exception as e:
Log.error(self, "Error while installing packages, "
"apt-get exited with error")
def auto_clean(self):
"""

Loading…
Cancel
Save