linspector/linspector-old
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

big change n parsing logic passing

Browse source
This commit is contained in:
Rafael.Timmerberg 2013-06-18 00:10:14 +02:00
commit 56906b9ebc
17 changed files with 570 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/config/config.py
View file

@ -1,7 +1,7 @@
import json
from tasks import parseTaskList
from members import parseMemberList
from periods import parsePeriodList
#from hostgroups import parseHostGroupList
#from layouts import parseLayoutList
@ -17,7 +17,7 @@ class Config:
self.tasks = parseTaskList(self.dict['tasks'])
self.members = parseMemberList(self.dict['members'], self.tasks, log)
self.periods = parsePeriodList(self.dict['periods'], log)
#self.periods = parsePeriodList(self.dict['periods'], log)
#self.hostgroups = parseHostGroupList(self.dict['hostgroups'],
# self.members,
# self.periods,

66
lib/config/hostgroups.py
View file

@ -1,3 +1,4 @@
class HostGroupException(Exception):
def __init__(self, msg):
self.msg = msg
@ -12,25 +13,80 @@ class HostGroupMissingArgumentException(HostGroupException):
class HostGroup:
def __init__(self, name, **kwargs):
self.name = name
tmp = "members"
self.members = []
if not tmp in kwargs:
raise HostGroupMissingArgumentException(tmp, name)
self.members = kwargs[tmp]
self.add_members(kwargs[tmp])
tmp = "hosts"
self.hosts = []
if not tmp in kwargs:
raise HostGroupMissingArgumentException(tmp, name)
self.hosts = kwargs[tmp]
self.add_hosts(kwargs[tmp])
tmp = "services"
if not tmp in kwargs:
raise HostGroupMissingArgumentException(tmp, name)
self.services = kwargs[tmp]
self.add_services(kwargs[tmp])
self.parents = []
tmp = "parents"
if tmp in kwargs:
self.add_parent(kwargs[tmp])
tmp = "processors"
self.processors = []
if tmp in kwargs:
self.add_processor(kwargs[tmp])
def _to_config_dict(self, configDict):
me = {}
me["members"] = [member.nameid for member in self.get_members()]
me["hosts"] = self.hosts
me["parents"] = [hg.get_name() for hg in self.get_parents()]
#TODO implement delegation
#me["services"] = [service._to_config_dict(configDict) for service in self.get_services()]
#me["processors"] = [processor._to_config_dict(configDict) for processor in self.get_processors()]
configDict["hostgroups"][self.get_name()] = me
def __add_internal(self,l,item):
if isinstance(item, list):
l.extend(item)
else:
l.append(item)
def add_members(self, member):
self.__add_internal(self.get_members(), member)
def add_hosts(self, host):
self.__add_internal(self.get_hosts(), host)
def add_processors(self, processor):
self.__add_internal(self.get_processors(), processor)
def add_parents(self, parent):
self.__add_internal(self.get_parents(), parent)
def get_parents(self):
return self.parents
def get_processors(self):
return self.processors
def get_services(self):
return self.services
def get_hosts(self):
return self.hosts
def get_name(self):
return self.name
def get_members(self):
return self.members
return self.members
def __str__(self):

9
lib/config/layouts.py
View file

@ -21,6 +21,15 @@ class Layout:
else:
self._members = members
def _to_config_dict(self, configDict):
me = {}
me["hostgroups"] = [hg.name for hg in self.get_hostgroups()]
me["enabled"] = self.is_enabled()
configDict["layouts"][self.get_name()] = me
for hostgroup in self.get_hostgroups():
hostgroup._to_config_dict(configDict)
def get_name(self):
return self._name

20
lib/config/members.py
View file

@ -2,19 +2,33 @@ import re
class Member:
def __init__(self, nameid, name="", phone="", comment="", parent="", filters=None):
self.nameid = nameid
def __init__(self, nameid, name="", phone="", comment="", parent="", tasks=None):
self.id = id
self.name = name
self.phone = phone
self.filters = filters
self.tasks = []
self.add_task(tasks)
self.comment = comment
self.parent = parent
def add_task(self, task):
if task is None:
return
if isinstance(task, list):
self.tasks.extend(task)
else:
self.tasks.append(task)
def get_tasks(self):
return self.tasks
def __str__(self):
ret = "Member Id: " + self.nameid + " Name: " + self.name + " Filters: " + str(self.phone)
for f in self.filters:
ret += str(f)
return ret
class MemberFilter:

184
lib/config/parser.py
View file

@ -1,9 +1,36 @@
from os.path import isfile
import json
import imp
import sys
from layouts import Layout
from hostgroups import HostGroup
from members import Member
from periods import CronPeriod, DatePeriod, IntervalPeriod
from lib.services.service import Service
from lib.processors.processor import Processor
from lib.parsers.parser import Parser
from lib.tasks.task import Task
from platform import processor
MOD_SERVICES = "services"
MOD_PROCESSORS = "processors"
MOD_PARSERS = "parsers"
MOD_TASKS = "tasks"
sys.path.append("../" + MOD_SERVICES)
sys.path.append("../" + MOD_PROCESSORS)
sys.path.append("../" + MOD_PARSERS)
sys.path.append("../" + MOD_TASKS)
KEY_LAYOUTS = "layouts"
KEY_HOSTGROUPS = "hostgroups"
KEY_MEMBERS = "members"
KEY_PERIODS = "periods"
KEY_CORE = "core"
class ConfigurationException(Exception):
def __init__(self, msg, log):
log.e(msg)
@ -13,11 +40,8 @@ class ConfigurationException(Exception):
return repr(self.msg)
KEY_LAYOUTS = "layouts"
KEY_HOSTGROUPS = "hostgroups"
KEY_MEMBERS = "members"
KEY_PERIODS = "periods"
KEY_CORE = "core"
class ConfigParser:
@ -32,6 +56,16 @@ class ConfigParser:
self.hostgroups = {}
self.members = {}
self.periods = {}
self.layouts = {}
self._loadedMods={MOD_SERVICES:{}, MOD_PROCESSORS:{}, MOD_TASKS:{}}
def _create_new_config_dict(self):
return {"members": {}, "periods":{}, "hostgroups":{}, "layouts":{}, "core":{}}
def create_config(self, config):
configDict = self._create_new_config_dict()
for layout in config.get_layouts():
layout._to_config_dict(configDict)
def _read_json_config(self, configFilename):
'''
@ -59,7 +93,18 @@ class ConfigParser:
returns the value as list
'''
return configValue if isinstance(configValue, list) else [configValue]
def _create_raw_Object(self, jsonDict, msgName, creator):
items = []
for key, val in jsonDict.items():
try:
item = creator(key, val)
items.append(item)
except Exception:
self.log.w("ignoring " + msgName + ": " + key + "! reason:")
self.log.w(str(Exception))
return items
def create_layouts_from_json(self, jsonLayouts):
layouts = []
for lName, lValues in jsonLayouts.items():
@ -98,11 +143,46 @@ class ConfigParser:
self.log.w("ignoring member: " + memberName + "!")
self.log.w("reason: " + str(Exception))
return members
def _load_module(self, clazz, modPart):
mods = self._loadedMods[modPart]
if clazz in mods:
return mods["class"]
else:
return __import__(clazz)
def replace_with_import(self, objList, modPart, items_func, class_check):
loadedModules = {}
for obj in objList:
repl = []
items = items_func(obj)
for clazzItem in items:
try:
clazz = clazzItem["class"]
mod = self._load_module(clazz, modPart)
item = mod.create(**clazzItem)
if class_check(item):
repl.append(item)
else:
self.log.w(" ignoring class " + clazzItem["class"] + "! It does not pass the class check!")
except ImportError, err:
self.log.w("could not import " + clazz + ": " + str(clazzItem) + "! reason")
self.log.w(str(err))
except KeyError:
self.log.w("Key 'class' not in classItem " + str(clazzItem))
except Exception:
self.log.w("Error while replace: " + str(Exception))
del items[:]
items.extend(repl)
def parse_config(self, configFilename):
'''
parses the json configuration and returns a list of layouts,
which contains all nessesary information of the config file.
It will only parse nessesary Objects.
Parsing will be done in 3 steps:
1. get raw Config Objects by just passing the values defined inside the config
2. replace references by objects
@ -115,6 +195,7 @@ class ConfigParser:
self.jsonDict = self._read_json_config(configFilename)
jsonLayouts = self.jsonDict[KEY_LAYOUTS]
#layouts = self._create_raw_Object(jsonLayouts, "Layouts", lambda name, vals: Layout(name, **vals))
layouts = self.create_layouts_from_json(jsonLayouts)
hostgroupNames = set()
@ -149,4 +230,93 @@ class ConfigParser:
jsonMembers[memberName] = self.jsonDict[KEY_HOSTGROUPS][memberName]
self.members = self.create_members_from_json(jsonMembers)
def parsePeriodList(name, values):
if "date" in values:
return DatePeriod(name, **values)
comp = ["weeks", "days", "hours", "minutes", "seconds", "start_date"]
if len([i for i in comp if i in values]) > 0 :
return IntervalPeriod(name, **values)
comp = ["year", "month", "day", "week", "day_of_week", "hour", "minute", "second"]
if len([i for i in comp if i in values]) > 0 :
return CronPeriod(name, **values)
else:
raise ConfigurationException("could not determine correct Period(" + repr(values)+").")
class FullConfigParser(ConfigParser):
def parse_config(self, configFilename):
'''
parses the json configuration and returns a list of layouts,
which contains all nessesary information of the config file.
parses the full config
Parsing will be done in 3 steps:
1. get raw Config Objects by just passing the values defined inside the config
2. replace references by objects, import services, tasks, parsers and processors
3. do sanity checks
params:
configFilename: indicates which configuration file to parse
'''
self.jsonDict = self._read_json_config(configFilename)
# first step
creator = lambda name, values: Layout(name,**values)
layouts = self._create_raw_Object(self.jsonDict[KEY_LAYOUTS], "Layout", creator)
creator = lambda name, values: Member(name, **values)
members = self._create_raw_Object(self.jsonDict[KEY_MEMBERS], "Member", creator)
creator = lambda name, values: HostGroup(name, **values)
hostgroups = self._create_raw_Object(self.jsonDict[KEY_HOSTGROUPS], "Hostgroup", creator)
creator = parsePeriodList
periods = self._create_raw_Object(self.jsonDict[KEY_PERIODS], "Period", creator)
#2. import and replace
items_func = lambda hostgroup: hostgroup.get_services()
class_check = lambda service: isinstance(service, Service)
self.replace_with_import(hostgroups, MOD_SERVICES, items_func, class_check)
items_func = lambda hostgroup: hostgroup.get_processors()
class_check = lambda processor: isinstance(processor, Processor)
self.replace_with_import(hostgroups, MOD_PROCESSORS, items_func, class_check)
items_func = lambda service: service.get_parser()
class_check = lambda parser: isinstance(parser, Parser)
self.replace_with_import(hostgroups.services, MOD_PARSERS, items_func, class_check)
items_func = lambda member: member.get_tasks()
class_check = lambda task: isinstance(task, Task)
self.replace_with_import(members, MOD_TASKS, items_func, class_check)
for hg in hostgroups:
replmembers = []
memberNames = hg.get_members()
for membername in memberNames:
member = [m for m in members if m.id == membername]
if len(member) == 1:
replmembers.append(member[0])
del hg.get_members()[:]
hg.add_members(replmembers)
replParents = []
parentNames = hg.get_parents()
for parentname in parentNames:
parent = [p for p in hostgroups if p.get_name() == parentname]
if len(parent) == 1:
replParents.append(parent[0])

23
lib/config/periods.py
View file

@ -69,26 +69,5 @@ class DatePeriod(Period):
return scheduler.add_date_job(func, self.date, jobInfo)
def parsePeriodList(periodlist, log):
periods = []
#log.d("values of periodslist: " + str(periodlist.items()))
for name, values in periodlist.items():
if "date" in values:
periods.append(DatePeriod(name, **values))
continue
comp = ["weeks", "days", "hours", "minutes", "seconds", "start_date"]
if len([i for i in comp if i in values]) > 0 :
periods.append(IntervalPeriod(name, **values))
break
comp = ["year", "month", "day", "week", "day_of_week", "hour", "minute", "second"]
if len([i for i in comp if i in values]) > 0 :
periods.append(CronPeriod(name, **values))
break
else:
log.w("ignoring Period: " + str(name))
log.w("reason: could not determine PeriodType: " + str(values))
return periods

9
lib/processors/mongodb.py
View file

@ -1,3 +1,10 @@
"""
The MongoDB processor
"""
"""
from processor import Processor
class Mongodb(Processor):
def __init__(self):
pass

54
lib/services/http.py
View file

@ -13,37 +13,43 @@ from service import Service
class HttpService(Service):
def __init__(self, parser, log, **kwargs):
super(Service, self).__init__(parser)
if "string" in kwargs:
self.string = kwargs["string"]
else:
log.w("There is no string set to match")
raise
if "method" in kwargs:
self.method = kwargs["method"]
else:
self.method = "get"
if "params" in kwargs:
def __init__(self, **kwargs):
super(HttpService, self).__init__(**kwargs)
args = self.get_arguments()
self.method = "get"
if "method" in args:
self.method = args["method"]
self.params = None
if "params" in args:
self.params = kwargs["params"]
if "path" in kwargs:
self.path = "/"
if "path" in args:
self.path = kwargs["path"]
else:
self.path = "/"
if "port" in kwargs:
self.port = "80"
if "port" in args:
self.port = kwargs["port"]
else:
self.port = "80"
if "protocol" in kwargs:
self.protocol = "http"
if "protocol" in args:
self.protocol = kwargs["protocol"]
else:
self.protocol = "http"
def needs_arguments(self):
return True
def execute(self):
params = urllib.urlencode(self.params)
if self.method is "get":
f = urllib.urlopen(self.protocol + "://" + self.host + ":" + self.port + self.path + "?%s" % params)
f = urllib.urlopen(self.protocol + "://" + self._host + ":" + self.port + self.path + "?%s" % params)
elif self.method is "post":
f = urllib.urlopen(self.protocol + "://" + self.host + ":" + self.port + self.path, params)
f = urllib.urlopen(self.protocol + "://" + self._host + ":" + self.port + self.path, params)
#print f.read()
#print f.read()
def create(**kwargs):
return HttpService(**kwargs)

11
lib/services/ping.py
View file

@ -2,4 +2,13 @@
The ping service in pure Python.
"""
# http://code.activestate.com/recipes/409689-icmplib-library-for-creating-and-reading-icmp-pack/
# http://code.activestate.com/recipes/409689-icmplib-library-for-creating-and-reading-icmp-pack/
from service import Service
class PingService(Service):
def __init__(self, **kwargs):
super(PingService, self).__init__(**kwargs)
def create(self, **kwargs):
return PingService(**kwargs)

105
lib/services/service.py
View file

@ -1,9 +1,103 @@
KEY_PARSER = "parser"
KEY_COMMENT = "comment"
KEY_THRESHOLD = "threshold"
KEY_FAILS = "fails"
KEY_PERIODS = "periods"
KEY_ARGS = "args"
class Service:
def __init__(self, host, parser):
self.host = host
self.parser = parser
def __init__(self, **kwargs):
self._args = {}
if KEY_ARGS in kwargs:
self.add_arguments(kwargs[KEY_ARGS])
elif self.needs_arguments():
raise Exception("Error: needs arguments but none provided!")
self._host = None
self._parser = []
if KEY_PARSER in kwargs:
self.add_parser(kwargs[KEY_PARSER])
self._comment = None
if KEY_COMMENT in kwargs:
self._comment = kwargs[KEY_COMMENT]
self._threshold = 0
if KEY_THRESHOLD in kwargs:
self._threshold = kwargs[KEY_THRESHOLD]
self._fails = {}
if KEY_FAILS in kwargs:
self.put_fails(kwargs[KEY_FAILS])
self._periods = []
if KEY_PERIODS in kwargs:
self.addPeriods(kwargs[KEY_PERIODS])
self.errorcode = 0
self.errormessage = "No Error!"
def add_arguments(self, args):
for key, val in args.items():
self._args[key] = val
def add_argument(self, key, value):
self._args[key] = value
def get_arguments(self):
return self._args
def add_period(self, period):
if period is not None:
if isinstance(period, list):
self._periods.extend(period)
else:
self._periods.append(period)
def get_periods(self):
return self._periods
def get_fails(self):
return self._fails
def has_fail(self, fail):
return fail in self.get_fails()
def put_fail(self, key, value):
self._fails[key] = value
def put_fails(self, fails):
for key, value in fails.items():
self.put_fail(key, value)
def get_threshold(self):
return self._threshold
def get_comment(self):
return self._comment
def set_host(self, host):
self._host = host
def get_host(self):
return self._host
def get_parser(self):
return self._parser
def add_parser(self, parser):
if parser is not None:
if isinstance(parser, list):
self._parser.extend(parser)
else:
self._parser.append(parser)
def needs_arguments(self):
return False
def _execute(self):
self.pre_execute()
@ -18,7 +112,10 @@ class Service:
pass
def parse_result(self, executionResult):
return self.parser._parse(executionResult)
result = []
for parser in self.get_parser():
result.append(self._parser._parse(executionResult))
def handle_result(self, parseResult):
pass

22
lib/services/shell.py
View file

@ -6,13 +6,21 @@ from service import Service
class ShellService(Service):
def __init__(self, parser, log, **kwargs):
super(Service, self).__init__(parser)
if "command" in kwargs:
self.command = kwargs["command"]
def __init__(self, **kwargs):
super(ShellService, self).__init__(**kwargs)
args = self.get_arguments()
if "command" in args:
self.command = args["command"]
else:
log.w("There is no command")
raise
raise Exception("There is no command argument")
def needs_arguments(self):
return True
def execute(self):
self.command.call()
self.command.call()
def create(**kwargs):
return ShellService(**kwargs)

77
lib/services/snmpget.py
View file

@ -2,45 +2,56 @@
The snmpget service in pure Python.
"""
from pysnmp.entity.rfc3413.oneliner import cmdgen
#from pysnmp.entity.rfc3413.oneliner import cmdgen
from service import Service
from Cython.Compiler.Naming import kwds_cname
from wx.lib.pubsub.core import kwargs
class SnmpgetService(Service):
def __init__(self, parser, log, **kwargs):
super(Service, self).__init__(parser)
if "community" in kwargs:
self.community = kwargs["community"]
def __init__(self, **kwargs):
super(SnmpgetService, self).__init__(**kwargs)
args = self.get_arguments()
if "community" in args:
self.community = args["community"]
else:
log.w("There is no community")
raise
if "oid" in kwargs:
self.oid = kwargs["oid"]
raise Exception("There is no community")
if "oid" in args:
self.oid = args["oid"]
else:
log.w("There is no oid")
raise
if "port" in kwargs:
self.port = kwargs["port"]
else:
self.port = "161"
raise Exception("There is no oid")
self.port = "161"
if "port" in args:
self.port = args["port"]
def needs_arguments(self):
return True
def execute(self):
cmdGen = cmdgen.CommandGenerator()
pass
#cmdGen = cmdgen.CommandGenerator()
errorIndication, errorStatus, errorIndex, varBinds = cmdGen.getCmd(
cmdgen.CommunityData(self.community),
cmdgen.UdpTransportTarget((self.host, self.port)),
cmdgen.MibVariable(self.oid)
)
if errorIndication:
print(errorIndication)
else:
if errorStatus:
print('%s at %s' % (
errorStatus.prettyPrint(),
errorIndex and varBinds[int(errorIndex) - 1] or '?'
))
else:
for name, val in varBinds:
print('%s = %s' % (name.prettyPrint(), val.prettyPrint()))
#errorIndication, errorStatus, errorIndex, varBinds = cmdGen.getCmd(
# cmdgen.CommunityData(self.community),
# cmdgen.UdpTransportTarget((self._host, self.port)),
# cmdgen.MibVariable(self.oid)
#)
#if errorIndication:
# print(errorIndication)
#else:
# if errorStatus:
# print('%s at %s' % (
# errorStatus.prettyPrint(),
# errorIndex and varBinds[int(errorIndex) - 1] or '?'
# ))
# else:
# for name, val in varBinds:
# print('%s = %s' % (name.prettyPrint(), val.prettyPrint()))
def create(**kargs):
return SnmpgetService(**kwargs)

22
lib/services/ssh.py
View file

@ -11,14 +11,18 @@ from service import Service
class SshService(Service):
def __init__(self, parser, log, **kwargs):
super(Service, self).__init__(parser)
if "command" in kwargs:
self.command = kwargs["command"]
def __init__(self, **kwargs):
super(SshService, self).__init__(**kwargs)
args = self.get_arguments()
if "command" in args:
self.command = args["command"]
else:
log.w("There is no command")
raise
raise Exception("There is no command argument")
def needs_arguments(self):
return True
def execute(self):
path = os.path.join(os.environ['HOME'], '.ssh', 'id_rsa')
key = paramiko.RSAKey.from_private_key_file(path)
@ -35,7 +39,9 @@ class SshService(Service):
print '... ' + line.strip('\n')
client.close()
def create(**kwargs):
return SshService(**kwargs)
# def main():
# # service = SshService(parser, log, command='uptime')
# return

22
lib/services/tcpconnect.py
View file

@ -10,13 +10,18 @@ from service import Service
class TcpconnectService(Service):
def __init__(self, parser, log, **kwargs):
super(Service, self).__init__(parser)
if "port" in kwargs:
self.port = kwargs["port"]
def __init__(self, **kwargs):
super(TcpconnectService, self).__init__(**kwargs)
args = self.get_arguments()
if "port" in args:
self.port = args["port"]
else:
log.w("There is no port set")
raise
raise Exception("There is no port set")
def needs_arguments(self):
return True
def execute(self, log):
try:
@ -32,4 +37,7 @@ class TcpconnectService(Service):
self.errorcode = 2
sock.close()
return
return
def create(**kwargs):
return TcpconnectService(**kwargs)

23
lib/tasks/email.py
View file

@ -1,3 +1,24 @@
"""
The email task.
"""
"""
from lib.tasks.task import Task
class EmailTask(Task):
def __init__(self, **kwargs):
if not "type" in kwargs:
raise Exception("'type' not in typeDict " + str(kwargs))
if not "args" in kwargs:
raise Exception("typeDict " + str(kwargs) + " has nor arguments!")
self.set_task_type(kwargs["type"])
self.recipient = kwargs["args"]["rcpt"]
def execute_task(self, msg):
pass
def creator(**taskDict):
return EmailTask(**taskDict)

23
lib/tasks/sms.py
View file

@ -1,3 +1,24 @@
"""
The sms task.
"""
"""
from task import Task
class SmsTask(Task):
def __init__(self, **kwargs):
if not "type" in kwargs:
raise Exception("'type' not in typeDict " + str(kwargs))
if not "args" in kwargs:
raise Exception("typeDict " + str(kwargs) + " has no arguments!")
self.set_task_type(kwargs["type"])
self.recipient = kwargs["args"]["rcpt"]
def execute_task(self, msg):
pass
def creator(**taskDict):
return SmsTask(**taskDict)

23
lib/tasks/xmpp.py
View file

@ -1,3 +1,24 @@
"""
The xmpp task.
"""
"""
from task import Task
class XmppTask(Task):
def __init__(self, **kwargs):
if not "type" in kwargs:
raise Exception("'type' not in typeDict " + str(kwargs))
if not "args" in kwargs:
raise Exception("typeDict " + str(kwargs) + " has nor arguments!")
self.set_task_type(kwargs["type"])
self.recipient = kwargs["args"]["rcpt"]
def execute_task(self, msg):
pass
def creator(**taskDict):
return XmppTask(taskDict)