File: //usr/local/qcloud/monitor/barad/comm/base_process.py
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import logging
from logging.handlers import RotatingFileHandler
import sys
import cutils
import os
import gzip
from contextlib import closing
class CompressedRotatingFileHandler(RotatingFileHandler):
CHUNK_SIZE = 64 * 1024 # 64KB chunks for better performance
def _safe_remove(self, filepath):
"""安全删除文件
Args:
filepath: 要删除的文件路径
"""
try:
if os.path.exists(filepath):
os.remove(filepath)
except (IOError, OSError) as e:
self.handleError("Error removing file %s: %s" % (filepath, str(e)))
def _compress_file(self, source_path, dest_path):
"""压缩文件
Args:
source_path: 源文件路径
dest_path: 目标文件路径
Returns:
bool: 压缩是否成功
"""
try:
with closing(open(source_path, 'rb')) as f_in:
with closing(gzip.open(dest_path, 'wb')) as f_out:
while True:
chunk = f_in.read(self.CHUNK_SIZE)
if not chunk:
break
f_out.write(chunk)
return True
except (IOError, OSError) as e:
self.handleError("Error compressing %s to %s: %s" % (source_path, dest_path, str(e)))
return False
def doRollover(self):
"""执行默认日志轮转"""
RotatingFileHandler.doRollover(self)
# 处理最后一个备份文件
last_backup = "%s.%d" % (self.baseFilename, self.backupCount)
self._safe_remove(last_backup + ".gz")
# 轮转现有的备份文件
for i in range(self.backupCount - 1, 0, -1):
sfn = "%s.%d.%s" % (self.baseFilename, i, "gz")
dfn = "%s.%d.%s" % (self.baseFilename, i + 1, "gz")
if os.path.exists(sfn):
self._safe_remove(dfn)
try:
os.rename(sfn, dfn)
except (IOError, OSError) as e:
self.handleError("Error renaming %s to %s: %s" % (sfn, dfn, str(e)))
# 压缩轮换后所有日志文件
for i in range(1, self.backupCount + 1):
log_file = "%s.%d" % (self.baseFilename, i)
if os.path.exists(log_file):
try:
# 压缩文件
if(self._compress_file(log_file,log_file + ".gz")):
# 压缩成功后删除原始文件
os.remove(log_file)
except (IOError, OSError) as e:
self.handleError("Error _compress_file %s: %s" % (log_file, str(e)))
class BaseProcess(object):
def __init__(self, config_path, section_name):
self.__config = cutils.generate_config(config_path, section_name)
self.__logger = None
log_config = cutils.generate_config(config_path, 'log')
if len(log_config) > 0:
self.__logger = self.__init_log(log_config['path'] + '/' + section_name + '.log', log_config['level'], int(log_config['size']), int(log_config['backup']))
else:
#if no log configure , write it to the console
print 'NOTICE!! no log configuration found, write log to stdout'
self.__logger = cutils.console_logger()
self.__logger.info("start %s , pid = %d" % (section_name, os.getpid()))
def __init_log(self, name, level, size, backup):
LEVELS = {'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL
}
if not LEVELS.has_key(level):
level = 'error'
logger = logging.getLogger()
common_log_h = CompressedRotatingFileHandler(filename=name,mode='a',maxBytes=size*1024*1024,backupCount=backup)
formatter_common = logging.Formatter('%(asctime)s %(levelname)s %(filename)s %(lineno)d %(message)s')
common_log_h.setFormatter(formatter_common)
logger.addHandler(common_log_h)
logger.setLevel(LEVELS[level])
return logger
def get_config(self, key):
return self.__config.get(key)
def logger(self):
return self.__logger