File: //usr/local/qcloud/monitor/barad/plugin/collector/vm/local.py
import os
import sys
import re
import time
from datetime import datetime
sys.path.append(os.getcwd() + '/../../../comm/')
import constant
exitfalg = 0
try:
import _sqlite3
except Exception as e :
exitfalg = 1
from plugin_base import VmBaseCollector
from utils.collect_tool import CpuCollect
from utils.metric_handler import MetricHandler
import cutils
from cutils import CommUtils
CONFIG_PATH = '../../../etc/config.ini'
class LocalCollector(VmBaseCollector):
def init(self):
self.set_frequency(10)
self.databasename = ("/usr/local/qcloud/monitor/barad/log/")
self.saveperiod = 7
self.lastgetconfig = 0
self.lastdbrote = 0
#table cols (use for insert)
self.vmstatcols = ""
self.netstatcols = ""
self.arpstatcols = ""
self.sockstatcols = ""
#collect period
self.vmstatexecperiod = 10
self.netstatexecperiod = 10
self.arpcacheexecperiod = 10
self.vmstatexecperiod = 10
#last collection time
self.vmstatlastexectime = 0
self.netstatlastexectime = 0
self.arpcachelastexectime = 0
self.sockstatlastexectime = 0
self.vmstatconf = ("vmstat", "/proc/vmstat", 10)
self.netstatconf = ("netstat", "/proc/net/netstat", 10)
self.arpcacheconf = ("arp_cache", "/proc/net/stat/arp_cache", 10)
self.socketconf = ("sockstat", "/proc/net/sockstat", 10)
def do_collect(self):
global exitfalg
if exitfalg == 1 :
return
now = int(time.time())
if self.lastgetconfig == 0 or now - self.lastgetconfig >= 60:
self.lastgetconfig = now
try :
localconf = cutils.generate_config(CONFIG_PATH,'localSerialization')
try :
self.databasename = localconf.get('dbpath').replace(' ','')
self.saveperiod = int(localconf.get('saveperiod').replace(' ',''))
except Exception as e:
pass
self.vmstatconf = tuple(localconf.get('vmstat').replace(' ','').split(','))
self.netstatconf = tuple(localconf.get('netstat').replace(' ','').split(','))
self.arpcacheconf = tuple(localconf.get('arpcache').replace(' ','').split(','))
self.socketconf = tuple(localconf.get('sockstat').replace(' ','').split(','))
try:
self.vmstatexecperiod = int(self.vmstatconf[2].strip())
except Exception as e:
pass
try:
self.netstatexecperiod = int(self.netstatconf[2].strip())
except Exception as e:
pass
try:
self.arpexecperiod = int(self.arpcacheconf[2].strip())
except Exception as e:
pass
try:
self.sockstatexecperiod = int(self.arpcacheconf[2].strip())
except Exception as e:
pass
except Exception as e:
pass
#update conf data per horus and upgrade database
self.vmstat(self.vmstatconf)
self.netstat(self.netstatconf)
self.arp_cache(self.arpcacheconf)
self.sockstat(self.socketconf)
if self.lastdbrote == 0 or now - self.lastdbrote >= 24*60*60:
self.dbrote(self.databasename, self.saveperiod)
self.lastdbrote = now
if self.vmstatlastexectime == 0 or now - self.vmstatlastexectime >= self.vmstatexecperiod :
self.vmstatlastexectime = now
self.savevmstat(self.vmstatconf)
if self.netstatlastexectime == 0 or now - self.netstatlastexectime >= self.netstatexecperiod :
self.netstatlastexectime = now
self.savenetstat(self.netstatconf)
if self.arpcachelastexectime == 0 or now - self.arpcachelastexectime >= self.arpcacheexecperiod :
self.arpcachelastexectime = now
self.savearp_cache(self.arpcacheconf)
if self.sockstatlastexectime == 0 or now - self.sockstatlastexectime >= self.sockstatexecperiod :
self.sockstatlastexectime = 0
self.savesockstat(self.socketconf)
def dbrote(self, dbpath, dbcount):
dbname = "record"
try :
# del all file when that file create 6 days ago
cmd = 'find ' + self.databasename + ' -ctime +' + str(int(dbcount)-1) + ' | egrep "[0-9]{8}_' + dbname + '.db" | xargs rm -rf'
CommUtils.ExecuteTimeoutCommand(cmd, 3).strip()
except Exception as e :
pass
def execsql(self, sql, parms = None ):
try :
if not os.path.exists(self.databasename):
os.makedirs(self.databasename,0700)
sdate = time.strftime('%Y%m%d')
conn = _sqlite3.connect(self.databasename + sdate+"_record.db")
if parms is not None:
conn.execute(sql, parms)
else:
conn.execute(sql)
conn.commit()
except Exception as e:
pass
finally :
conn.close()
def savevmstat(self, info = ("vmstat", "/proc/vmstat")):
strsql=""
data = []
tablename = "VMSTAT"
filename = "/proc/vmstat"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
try :
if os.path.exists(filename):
with open(filename, 'r') as f:
for line in f.readlines():
if strsql != "":
strsql = strsql + ", ?"
else :
strsql = strsql + "?"
data.append(int(line.split()[1].strip()))
self.execsql("INSERT INTO " + tablename + " ( " + self.vmstatcols + " ) VALUES ( " + strsql + " );", tuple(data))
except Exception as e:
pass
def savenetstat(self, info = ("netstat", "/proc/net/netstat")):
strsql = ""
data = []
tablename = "NETSTAT"
filename = "/proc/net/netstat"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
pattern = re.compile("TcpExt: [0-9]{1,10}")
try :
if os.path.exists(filename):
with open(filename, 'r') as f:
for line in f.readlines():
if pattern.match(line) is not None:
col = line.split()
if len(col) > 0 :
col = col[1:len(col)]
for item in col:
if strsql != "":
strsql = strsql + ", ?"
else:
strsql = strsql + "?"
data.append(int(item))
pattern = re.compile("IpExt: [0-9]{1,10}")
if os.path.exists(filename):
with open(filename, 'r') as f:
for line in f.readlines():
if pattern.match(line) is not None:
col = line.split()
if len(col) > 0 :
col = col[1:len(col)]
for item in col:
if strsql != "":
strsql = strsql + ", ?"
else:
strsql = strsql + "?"
data.append(int(item))
self.execsql("INSERT INTO " + tablename + " ( " + self.netstatcols + " ) values ( " + strsql + " );", tuple(data))
except Exception as e:
pass
def savearp_cache(self, info = ("arp_cache", "/proc/net/stat/arp_cache")):
strsql = ""
data = []
tablename = "ARP_CACHE"
filename = "/proc/net/stat/arp_cache"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
try :
val = self.get_stat(filename)
for key in val.keys():
if strsql != "":
strsql = strsql + ", ?"
else:
strsql = strsql + "?"
data.append(int(val.get(key,0)))
self.execsql("INSERT INTO " + tablename + " ( " + self.arpstatcols + " ) values ( " + strsql + " );", tuple(data))
except Exception as e:
pass
def savesockstat(self, info = ("sockstat", "/proc/net/sockstat")):
strsql=""
data = []
tablename = "SOCKSTAT"
filename = "/proc/net/sockstat"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
try :
if os.path.exists(info[1]):
with open(info[1], 'r') as f:
for line in f.readlines():
head = line.strip().split(':')
value = head[1].strip().split()
for i in range(1,len(value),2):
if strsql != "":
strsql = strsql + ", ?"
else :
strsql = strsql + "?"
data.append(int(value[i].strip()))
self.execsql("INSERT INTO " + tablename + " ( " + self.sockstatcols + " ) VALUES ( " + strsql + " );", tuple(data))
except Exception as e:
pass
def vmstat(self, info = ("vmstat", "/proc/vmstat")):
self.vmstatcols = ""
strsql=""
tablename = "VMSTAT"
filename = "/proc/vmstat"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
try :
if os.path.exists(filename):
with open(filename, 'r') as f:
for line in f.readlines():
strsql = strsql + line.split()[0].strip() + " INT NOT NULL,"
if self.vmstatcols == "":
self.vmstatcols = self.vmstatcols + line.split()[0].strip()
else:
self.vmstatcols = self.vmstatcols + ", "+ line.split()[0].strip()
self.execsql('''CREATE TABLE IF NOT EXISTS ''' + tablename + '''(ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,'''+strsql+''' CREATETIME DATETIME DEFAULT (strftime('%Y-%m-%d %H:%M:%S','now','localtime')));''')
except Exception as e:
pass
def netstat(self, info = ("netstat", "/proc/net/netstat")):
self.netstatcols = ""
tablename = "NETSTAT"
filename = "/proc/net/netstat"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
strsql=""
pattern = re.compile("TcpExt: [a-zA-Z]{1,8}")
col=[]
try :
if os.path.exists(filename):
with open(filename, 'r') as f:
for line in f.readlines():
if pattern.match(line) is not None:
col = line.split()
if len(col) > 0 :
col = col[1:len(col)]
for item in col:
strsql = strsql + item.strip() + " INT NOT NULL,"
if self.netstatcols == "":
self.netstatcols = self.netstatcols + item.strip()
else:
self.netstatcols = self.netstatcols + ", "+ item.strip()
pattern = re.compile("IpExt: [a-zA-Z]{1,8}")
if os.path.exists(filename):
with open(filename, 'r') as f:
for line in f.readlines():
if pattern.match(line) is not None:
col = line.split()
if len(col) > 0 :
col = col[1:len(col)]
for item in col:
strsql = strsql + item.strip() + " INT NOT NULL,"
if self.netstatcols == "":
self.netstatcols = self.netstatcols + item.strip()
else:
self.netstatcols = self.netstatcols + ", "+ item.strip()
self.execsql('''CREATE TABLE IF NOT EXISTS ''' + tablename + '''(ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,'''+strsql+''' CREATETIME DATETIME DEFAULT (strftime('%Y-%m-%d %H:%M:%S','now','localtime')));''')
except Exception as e:
pass
def arp_cache(self, info = ("arp_cache", "/proc/net/stat/arp_cache")):
self.arpstatcols = ""
tablename = "ARP_CACHE"
filename = "/proc/net/stat/arp_cache"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
strsql=""
col = []
val = self.get_stat(filename)
for key in val.keys():
strsql = strsql + key + " INT NOT NULL, "
if self.arpstatcols == "":
self.arpstatcols = self.arpstatcols + key
else:
self.arpstatcols = self.arpstatcols + ", "+ key
try :
self.execsql('''CREATE TABLE IF NOT EXISTS ''' + tablename + '''(ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,'''+strsql+''' CREATETIME DATETIME DEFAULT (strftime('%Y-%m-%d %H:%M:%S','now','localtime')));''')
except Exception as e:
pass
def get_stat(self, path):
arr = []
title = []
return_val = {}
val_list = []
try :
arr=[]
if os.path.exists(path):
with open(path,"r") as file :
lines = file.readlines()
title = lines[0].strip().split()
for line in lines[1::] :
arr.append(line.strip().split())
arr = [[ int(val,16) for val in lt ] for lt in arr]
val_list = [sum(x) for x in zip(*arr)]
except Exception as e:
pass
return dict(zip(title, val_list))
def sockstat(self, info = ("sockstat", "/proc/net/sockstat")):
self.sockstatcols = ""
strsql=""
tablename = "SOCKSTAT"
filename = "/proc/net/sockstat"
try :
tablename = info[0].strip().upper()
except Exception as e:
pass
try :
filename = info[1].strip()
except Exception as e:
pass
try :
if os.path.exists(filename):
with open(filename, 'r') as f:
for line in f.readlines():
head = line.strip().split(':')
value = head[1].strip().split()
for i in range(0,len(value),2):
strsql = strsql + head[0].lower().strip() + '_' + value[i].lower().strip() + " INT NOT NULL,"
if self.sockstatcols == "":
self.sockstatcols = self.sockstatcols + head[0].lower().strip() + '_' + value[i].lower().strip()
else:
self.sockstatcols = self.sockstatcols + ", " + head[0].lower().strip() + '_' + value[i].lower().strip()
self.execsql('''CREATE TABLE IF NOT EXISTS ''' + tablename + '''(ID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,'''+strsql+''' CREATETIME DATETIME DEFAULT (strftime('%Y-%m-%d %H:%M:%S','now','localtime')));''')
except Exception as e:
pass
def main():
collector = LocalCollector()
while True :
collector.do_collect()
time.sleep (10)
if __name__ == '__main__':
main()