diff --git a/ee/cli/plugins/stack.py b/ee/cli/plugins/stack.py index cc9b90c2..96861825 100644 --- a/ee/cli/plugins/stack.py +++ b/ee/cli/plugins/stack.py @@ -9,6 +9,7 @@ from ee.core.shellexec import EEShellExec from ee.core.fileutils import EEFileUtils from ee.core.apt_repo import EERepo from ee.core.extract import EEExtract +from ee.core.mysql import EEMysql from pynginxconfig import NginxConfig import random import string @@ -55,8 +56,6 @@ class EEStackController(CementBaseController): dict(help='Install Adminer stack', action='store_true')), (['--utils'], dict(help='Install Utils stack', action='store_true')), - (['--anemometer'], - dict(help='Install Utils stack', action='store_true')), ] @expose(hide=True) @@ -207,6 +206,28 @@ class EEStackController(CementBaseController): os.makedirs('/var/www/22222/htdocs/db/') shutil.move('/tmp/Anemometer-master', '/var/www/22222/htdocs/db/anemometer') + chars = ''.join(random.sample(string.ascii_letters, 8)) + anemometer_db = EEMysql() + EEShellExec.cmd_exec('mysql < /var/www/22222/htdocs/db' + '/anemometer/install.sql') + anemometer_db.execute('grant select on *.* to \'anemometer\'' + '@\'localhost\'') + anemometer_db.execute('grant all on slow_query_log.* to' + '\'anemometer\'@\'localhost\' IDENTIFIED' + ' BY \''+chars+'\'') + anemometer_db.close() + # Custom Anemometer configuration + data = dict(host='localhost', port='3306', user='anemometer', + password=chars) + ee_anemometer = open('/var/www/22222/htdocs/db/anemometer' + '/conf/config.inc.php', 'w') + self.app.render((data), 'anemometer.mustache', + out=ee_anemometer) + ee_anemometer.close() + + if any('/usr/bin/pt-query-advisor' == x[1] + for x in packages): + EEShellExec.cmd_exec("chmod +x /usr/bin/pt-query-advisor") pass @expose() @@ -269,12 +290,16 @@ class EEStackController(CementBaseController): "opcache/ocp.php"], ["https://github.com/jokkedk/webgrind/" "archive/master.tar.gz", - '/tmp/webgrind.tar.gz'] - ] - if self.app.pargs.anemometer: - packages = packages + [["https://github.com/box/Anemometer/archive" + '/tmp/webgrind.tar.gz'], + ["http://bazaar.launchpad.net/~percona-too" + "lkit-dev/percona-toolkit/2.1/download/he" + "ad:/ptquerydigest-20110624220137-or26tn4" + "expb9ul2a-16/pt-query-digest", + "/usr/bin/pt-query-advisor"], + ["https://github.com/box/Anemometer/archive" "/master.tar.gz", - '/tmp/anemometer.tar.gz']] + '/tmp/anemometer.tar.gz'] + ] self.pre_pref(apt_packages) if len(apt_packages): diff --git a/ee/cli/templates/anemometer.mustache b/ee/cli/templates/anemometer.mustache new file mode 100644 index 00000000..e082ce50 --- /dev/null +++ b/ee/cli/templates/anemometer.mustache @@ -0,0 +1,255 @@ + '{{host}}', + 'port' => '{{port}}', + 'db' => 'slow_query_log', + 'user' => '{{user}}', + 'password' => '{{password}}', + 'tables' => array( + 'global_query_review' => 'fact', + 'global_query_review_history' => 'dimension' + ), + 'source_type' => 'slow_query_log' +); + +$conf['default_report_action'] = 'report'; + +$conf['reviewers'] = array( 'dba1','dba2'); +$conf['review_types'] = array( 'good', 'bad', 'ticket-created', 'needs-fix', 'fixed', 'needs-analysis', 'review-again'); + +$conf['history_defaults'] = array( + 'output' => 'table', + 'fact-group' => 'date', + 'fact-order' => 'date DESC', + 'fact-limit' => '90', + 'dimension-ts_min_start' => date("Y-m-d H:i:s", strtotime( '-90 day')), + 'dimension-ts_min_end' => date("Y-m-d H:i:s"), + 'table_fields' => array('date', 'index_ratio','query_time_avg','rows_sent_avg','ts_cnt','Query_time_sum','Lock_time_sum','Rows_sent_sum','Rows_examined_sum','Tmp_table_sum','Filesort_sum','Full_scan_sum') +); + +$conf['report_defaults'] = array( + 'fact-group' => 'checksum', + 'fact-order' => 'Query_time_sum DESC', + 'fact-limit' => '20', + 'dimension-ts_min_start' => date("Y-m-d H:i:s", strtotime( '-1 day')), + 'dimension-ts_min_end' => date("Y-m-d H:i:s"), + 'table_fields' => array('checksum','snippet', 'index_ratio','query_time_avg','rows_sent_avg','ts_cnt','Query_time_sum','Lock_time_sum','Rows_sent_sum','Rows_examined_sum','Tmp_table_sum','Filesort_sum','Full_scan_sum'), + 'dimension-pivot-hostname_max' => null +); + +$conf['graph_defaults'] = array( + 'fact-group' => 'minute_ts', + 'fact-order' => 'minute_ts', + 'fact-limit' => '', + 'dimension-ts_min_start' => date("Y-m-d H:i:s", strtotime( '-7 day')), + 'dimension-ts_min_end' => date("Y-m-d H:i:s"), + 'table_fields' => array('minute_ts'), + 'plot_field' => 'Query_time_sum', +); + +$conf['report_defaults']['performance_schema'] = array( + 'fact-order' => 'SUM_TIMER_WAIT DESC', + 'fact-limit' => '20', + 'fact-group' => 'DIGEST', + 'table_fields' => array( 'DIGEST', 'snippet', 'index_ratio', 'COUNT_STAR', 'SUM_TIMER_WAIT', 'SUM_LOCK_TIME','SUM_ROWS_AFFECTED','SUM_ROWS_SENT','SUM_ROWS_EXAMINED','SUM_CREATED_TMP_TABLES','SUM_SORT_SCAN','SUM_NO_INDEX_USED' ) +); + +$conf['history_defaults']['performance_schema'] = array( + 'fact-order' => 'SUM_TIMER_WAIT DESC', + 'fact-limit' => '20', + 'fact-group' => 'DIGEST', + 'table_fields' => array( 'DIGEST', 'index_ratio', 'COUNT_STAR', 'SUM_LOCK_TIME','SUM_ROWS_AFFECTED','SUM_ROWS_SENT','SUM_ROWS_EXAMINED','SUM_CREATED_TMP_TABLES','SUM_SORT_SCAN','SUM_NO_INDEX_USED' ) +); + +$conf['report_defaults']['performance_schema_history'] = array( + 'fact-group' => 'DIGEST', + 'fact-order' => 'SUM_TIMER_WAIT DESC', + 'fact-limit' => '20', + 'dimension-FIRST_SEEN_start' => date("Y-m-d H:i:s", strtotime( '-1 day')), + 'dimension-FIRST_SEEN_end' => date("Y-m-d H:i:s"), + 'table_fields' => array( 'DIGEST', 'snippet', 'index_ratio', 'COUNT_STAR', 'SUM_LOCK_TIME','SUM_ROWS_AFFECTED','SUM_ROWS_SENT','SUM_ROWS_EXAMINED','SUM_CREATED_TMP_TABLES','SUM_SORT_SCAN','SUM_NO_INDEX_USED' ) +); + +$conf['graph_defaults']['performance_schema_history'] = array( + 'fact-group' => 'minute_ts', + 'fact-order' => 'minute_ts', + 'fact-limit' => '', + 'dimension-FIRST_SEEN_start' => date("Y-m-d H:i:s", strtotime( '-7 day')), + 'dimension-FIRST_SEEN_end' => date("Y-m-d H:i:s"), + 'table_fields' => array('minute_ts'), + 'plot_field' => 'SUM_TIMER_WAIT', + 'dimension-pivot-hostname_max' => null +); + +$conf['history_defaults']['performance_schema_history'] = array( + 'output' => 'table', + 'fact-group' => 'date', + 'fact-order' => 'date DESC', + 'fact-limit' => '90', + 'dimension-FIRST_SEEN_start' => date("Y-m-d H:i:s", strtotime( '-90 day')), + 'dimension-FIRST_SEEN_end' => date("Y-m-d H:i:s"), + 'table_fields' => array( 'date', 'snippet', 'index_ratio', 'COUNT_STAR', 'SUM_LOCK_TIME','SUM_ROWS_AFFECTED','SUM_ROWS_SENT','SUM_ROWS_EXAMINED','SUM_CREATED_TMP_TABLES','SUM_SORT_SCAN','SUM_NO_INDEX_USED' ) +); +$conf['plugins'] = array( + + 'visual_explain' => '/usr/bin/pt-visual-explain', + 'query_advisor' => '/usr/bin/pt-query-advisor', + + 'show_create' => true, + 'show_status' => true, + + 'explain' => function ($sample) { + $conn = array(); + + if (!array_key_exists('hostname_max',$sample) or strlen($sample['hostname_max']) < 5) + { + return; + } + + $pos = strpos($sample['hostname_max'], ':'); + if ($pos === false) + { + $conn['port'] = 3306; + $conn['host'] = $sample['hostname_max']; + } + else + { + $parts = preg_split("/:/", $sample['hostname_max']); + $conn['host'] = $parts[0]; + $conn['port'] = $parts[1]; + } + + $conn['db'] = 'mysql'; + if ($sample['db_max'] != '') + { + $conn['db'] = $sample['db_max']; + } + + $conn['user'] = 'root'; + $conn['password'] = ''; + + return $conn; + }, +); + +$conf['reports']['slow_query_log'] = array( + 'join' => array ( + 'dimension' => 'USING (`checksum`)' + ), + 'fields' => array( + 'fact' => array( + 'group' => 'group', + 'order' => 'order', + 'having' => 'having', + 'limit' => 'limit', + 'first_seen'=> 'clear|reldate|ge|where', + 'where' => 'raw_where', + 'sample' => 'clear|like|where', + 'checksum' => 'clear|where', + 'reviewed_status' => 'clear|where', + + ), + + 'dimension' => array( + 'extra_fields' => 'where', + 'hostname_max' => 'clear|where', + 'ts_min' => 'date_range|reldate|clear|where', + 'pivot-hostname_max' => 'clear|pivot|select', + 'pivot-checksum' => 'clear|pivot|select', + ), + ), + 'custom_fields' => array( + 'checksum' => 'checksum', + 'date' => 'DATE(ts_min)', + 'hour' => 'substring(ts_min,1,13)', + 'hour_ts' => 'round(unix_timestamp(substring(ts_min,1,13)))', + 'minute_ts' => 'round(unix_timestamp(substring(ts_min,1,16)))', + 'minute' => 'substring(ts_min,1,16)', + 'snippet' => 'LEFT(dimension.sample,20)', + 'index_ratio' =>'ROUND(SUM(Rows_examined_sum)/SUM(rows_sent_sum),2)', + 'query_time_avg' => 'SUM(Query_time_sum) / SUM(ts_cnt)', + 'rows_sent_avg' => 'ROUND(SUM(Rows_sent_sum)/SUM(ts_cnt),0)', + ), + + 'callbacks' => array( + 'table' => array( + 'date' => function ($x) { $type=''; if ( date('N',strtotime($x)) >= 6) { $type = 'weekend'; } return array($x,$type); }, + 'checksum' => function ($x) { return array(dec2hex($x), ''); } + ) + ) + +); + +$conf['reports']['performance_schema'] = array( + 'fields' => array( + 'fact' => array( + 'order' => 'order', + 'having' => 'having', + 'limit' => 'limit', + 'first_seen' => 'date_range|reldate|clear|where', + 'where' => 'raw_where', + 'DIGEST' => 'clear|where', + 'DIGEST_TEXT' => 'clear|like|where', + 'group' => 'group', + ), + ), + 'custom_fields' => array( + 'snippet' => 'LEFT(fact.DIGEST_TEXT,20)', + 'index_ratio' =>'ROUND(SUM_ROWS_EXAMINED/SUM_ROWS_SENT,2)', + 'rows_sent_avg' => 'ROUND(SUM_ROWS_SENT/COUNT_STAR,0)', + + ), + + 'special_field_names' => array( + 'time' => 'FIRST_SEEN', + 'checksum' => 'DIGEST', + 'sample' => 'DIGEST_TEXT', + 'fingerprint' => 'DIGEST_TEXT', + ), +); + +$conf['reports']['performance_schema_history'] = array( + 'join' => array ( + 'dimension' => 'USING (`DIGEST`)' + ), + 'fields' => array( + 'fact' => array( + 'group' => 'group', + 'order' => 'order', + 'having' => 'having', + 'limit' => 'limit', + 'first_seen'=> 'clear|reldate|ge|where', + 'where' => 'raw_where', + 'DIGEST_TEXT' => 'clear|like|where', + 'DIGEST' => 'clear|where', + 'reviewed_status' => 'clear|where', + + ), + + 'dimension' => array( + 'extra_fields' => 'where', + 'hostname' => 'clear|where', + 'FIRST_SEEN' => 'date_range|reldate|clear|where', + 'pivot-hostname' => 'clear|pivot|select', + ), + ), + 'custom_fields' => array( + 'date' => 'DATE(fact.FIRST_SEEN)', + 'snippet' => 'LEFT(fact.DIGEST_TEXT,20)', + 'index_ratio' =>'ROUND(SUM_ROWS_EXAMINED/SUM_ROWS_SENT,2)', + 'rows_sent_avg' => 'ROUND(SUM_ROWS_SENT/COUNT_STAR,0)', + 'hour' => 'substring(dimension.FIRST_SEEN,1,13)', + 'hour_ts' => 'round(unix_timestamp(substring(dimension.FIRST_SEEN,1,13)))', + 'minute_ts' => 'round(unix_timestamp(substring(dimension.FIRST_SEEN,1,16)))', + 'minute' => 'substring(dimension.FIRST_SEEN,1,16)', + ), + + 'special_field_names' => array( + 'time' => 'FIRST_SEEN', + 'checksum' => 'DIGEST', + 'hostname' => 'hostname', + 'sample' => 'DIGEST_TEXT' + ), +); + +?> diff --git a/ee/core/mysql.py b/ee/core/mysql.py index 88598ce6..e2b80922 100644 --- a/ee/core/mysql.py +++ b/ee/core/mysql.py @@ -38,3 +38,8 @@ class EEMysql(): except Exception as e: print("Error occured while executing "+statement) return False + + def close(self): + self.cur.close() + self.conn.close() +