You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
499 lines
19 KiB
Python
499 lines
19 KiB
Python
1 year ago
|
############################################################################
|
||
|
|
# Copyright (C) gempa GmbH #
|
||
|
|
# All rights reserved. #
|
||
|
|
# Contact: gempa GmbH (seiscomp-dev@gempa.de) #
|
||
|
|
# #
|
||
|
|
# GNU Affero General Public License Usage #
|
||
|
|
# This file may be used under the terms of the GNU Affero #
|
||
|
|
# Public License version 3.0 as published by the Free Software Foundation #
|
||
|
|
# and appearing in the file LICENSE included in the packaging of this #
|
||
|
|
# file. Please review the following information to ensure the GNU Affero #
|
||
|
|
# Public License version 3.0 requirements will be met: #
|
||
|
|
# https://www.gnu.org/licenses/agpl-3.0.html. #
|
||
|
|
# #
|
||
|
|
# Other Usage #
|
||
|
|
# Alternatively, this file may be used in accordance with the terms and #
|
||
|
|
# conditions contained in a signed written agreement between you and #
|
||
|
|
# gempa GmbH. #
|
||
|
|
############################################################################
|
||
|
|
|
||
|
|
import os, time, sys
|
||
|
|
import seiscomp.core, seiscomp.client, seiscomp.datamodel
|
||
|
|
import seiscomp.io, seiscomp.system
|
||
|
|
|
||
|
|
|
||
|
|
def collectParams(container):
|
||
|
|
params = {}
|
||
|
|
for i in range(container.groupCount()):
|
||
|
|
params.update(collectParams(container.group(i)))
|
||
|
|
for i in range(container.structureCount()):
|
||
|
|
params.update(collectParams(container.structure(i)))
|
||
|
|
for i in range(container.parameterCount()):
|
||
|
|
p = container.parameter(i)
|
||
|
|
if p.symbol.stage == seiscomp.system.Environment.CS_UNDEFINED:
|
||
|
|
continue
|
||
|
|
params[p.variableName] = ",".join(p.symbol.values)
|
||
|
|
|
||
|
|
return params
|
||
|
|
|
||
|
|
|
||
|
|
def collect(idset, paramSetID):
|
||
|
|
paramSet = seiscomp.datamodel.ParameterSet.Find(paramSetID)
|
||
|
|
if not paramSet:
|
||
|
|
return
|
||
|
|
idset[paramSet.publicID()] = 1
|
||
|
|
if not paramSet.baseID():
|
||
|
|
return
|
||
|
|
collect(idset, paramSet.baseID())
|
||
|
|
|
||
|
|
|
||
|
|
def sync(paramSet, params):
|
||
|
|
obsoleteParams = []
|
||
|
|
seenParams = {}
|
||
|
|
i = 0
|
||
|
|
while i < paramSet.parameterCount():
|
||
|
|
p = paramSet.parameter(i)
|
||
|
|
if p.name() in params:
|
||
|
|
if p.name() in seenParams:
|
||
|
|
# Multiple parameter definitions with same name
|
||
|
|
sys.stderr.write(
|
||
|
|
"- %s:%s / duplicate parameter name\n" % (p.publicID(), p.name()))
|
||
|
|
p.detach()
|
||
|
|
continue
|
||
|
|
seenParams[p.name()] = 1
|
||
|
|
val = params[p.name()]
|
||
|
|
if val != p.value():
|
||
|
|
p.setValue(val)
|
||
|
|
p.update()
|
||
|
|
else:
|
||
|
|
obsoleteParams.append(p)
|
||
|
|
i = i + 1
|
||
|
|
|
||
|
|
for p in obsoleteParams:
|
||
|
|
p.detach()
|
||
|
|
|
||
|
|
for key, val in list(params.items()):
|
||
|
|
if key in seenParams:
|
||
|
|
continue
|
||
|
|
p = seiscomp.datamodel.Parameter.Create()
|
||
|
|
p.setName(key)
|
||
|
|
p.setValue(val)
|
||
|
|
paramSet.add(p)
|
||
|
|
|
||
|
|
|
||
|
|
class ConfigDBUpdater(seiscomp.client.Application):
|
||
|
|
def __init__(self, argc, argv):
|
||
|
|
seiscomp.client.Application.__init__(self, argc, argv)
|
||
|
|
self.setLoggingToStdErr(True)
|
||
|
|
self.setMessagingEnabled(True)
|
||
|
|
self.setDatabaseEnabled(True, True)
|
||
|
|
self.setAutoApplyNotifierEnabled(False)
|
||
|
|
self.setInterpretNotifierEnabled(False)
|
||
|
|
self.setMessagingUsername("_sccfgupd_")
|
||
|
|
self.setLoadConfigModuleEnabled(True)
|
||
|
|
# Load all configuration modules
|
||
|
|
self.setConfigModuleName("")
|
||
|
|
self.setPrimaryMessagingGroup(
|
||
|
|
seiscomp.client.Protocol.LISTENER_GROUP)
|
||
|
|
|
||
|
|
self._outputFile = ""
|
||
|
|
self._keyDir = None
|
||
|
|
|
||
|
|
def createCommandLineDescription(self):
|
||
|
|
self.commandline().addGroup("Input")
|
||
|
|
self.commandline().addStringOption("Input", "key-dir",
|
||
|
|
"Overrides the location of the default key directory ($SEISCOMP_ROOT/etc/key)")
|
||
|
|
self.commandline().addGroup("Output")
|
||
|
|
self.commandline().addStringOption("Output", "output,o",
|
||
|
|
"If given, an output XML file is generated")
|
||
|
|
|
||
|
|
def validateParameters(self):
|
||
|
|
if not seiscomp.client.Application.validateParameters(self):
|
||
|
|
return False
|
||
|
|
|
||
|
|
try:
|
||
|
|
self._outputFile = self.commandline().optionString("output")
|
||
|
|
# Switch to offline mode
|
||
|
|
self.setMessagingEnabled(False)
|
||
|
|
self.setDatabaseEnabled(False, False)
|
||
|
|
self.setLoadConfigModuleEnabled(False)
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
try:
|
||
|
|
self._keyDir = self.commandline().optionString("key-dir")
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def init(self):
|
||
|
|
if not seiscomp.client.Application.init(self):
|
||
|
|
return False
|
||
|
|
|
||
|
|
# Initialize the basic directories
|
||
|
|
filebase = seiscomp.system.Environment.Instance().installDir()
|
||
|
|
descdir = os.path.join(filebase, "etc", "descriptions")
|
||
|
|
|
||
|
|
# Load definitions of the configuration schema
|
||
|
|
defs = seiscomp.system.SchemaDefinitions()
|
||
|
|
if not defs.load(descdir):
|
||
|
|
sys.stderr.write("Error: could not read descriptions\n")
|
||
|
|
return False
|
||
|
|
|
||
|
|
if defs.moduleCount() == 0:
|
||
|
|
sys.stderr.write("Warning: no modules defined, nothing to do\n")
|
||
|
|
return False
|
||
|
|
|
||
|
|
# Create a model from the schema and read its configuration including
|
||
|
|
# all bindings.
|
||
|
|
model = seiscomp.system.Model()
|
||
|
|
if self._keyDir:
|
||
|
|
model.keyDirOverride = self._keyDir
|
||
|
|
model.create(defs)
|
||
|
|
model.readConfig()
|
||
|
|
|
||
|
|
# Find all binding mods for trunk. Bindings of modules where standalone
|
||
|
|
# is set to true are ignored. They are supposed to handle their bindings
|
||
|
|
# on their own.
|
||
|
|
self.bindingMods = []
|
||
|
|
for i in range(defs.moduleCount()):
|
||
|
|
mod = defs.module(i)
|
||
|
|
# Ignore stand alone modules (eg seedlink, slarchive, ...) as they
|
||
|
|
# are not using the trunk libraries and don't need database
|
||
|
|
# configurations
|
||
|
|
if mod.isStandalone():
|
||
|
|
continue
|
||
|
|
self.bindingMods.append(mod.name)
|
||
|
|
|
||
|
|
if len(self.bindingMods) == 0:
|
||
|
|
sys.stderr.write(
|
||
|
|
"Warning: no usable modules found, nothing to do\n")
|
||
|
|
return False
|
||
|
|
|
||
|
|
self.stationSetups = {}
|
||
|
|
|
||
|
|
# Read bindings
|
||
|
|
for m in self.bindingMods:
|
||
|
|
mod = model.module(m)
|
||
|
|
if not mod:
|
||
|
|
sys.stderr.write("Warning: module %s not assigned\n" % m)
|
||
|
|
continue
|
||
|
|
if len(mod.bindings) == 0:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Rename global to default for being compatible with older
|
||
|
|
# releases
|
||
|
|
if m == "global":
|
||
|
|
m = "default"
|
||
|
|
|
||
|
|
sys.stderr.write("+ %s\n" % m)
|
||
|
|
for staid in list(mod.bindings.keys()):
|
||
|
|
binding = mod.getBinding(staid)
|
||
|
|
if not binding:
|
||
|
|
continue
|
||
|
|
# sys.stderr.write(" + %s.%s\n" % (staid.networkCode, staid.stationCode))
|
||
|
|
params = {}
|
||
|
|
for i in range(binding.sectionCount()):
|
||
|
|
params.update(collectParams(binding.section(i)))
|
||
|
|
key = (staid.networkCode, staid.stationCode)
|
||
|
|
if not key in self.stationSetups:
|
||
|
|
self.stationSetups[key] = {}
|
||
|
|
self.stationSetups[key][m] = params
|
||
|
|
sys.stderr.write(" + read %d stations\n" %
|
||
|
|
len(list(mod.bindings.keys())))
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def printUsage(self):
|
||
|
|
|
||
|
|
print('''Usage:
|
||
|
|
bindings2cfg [options]
|
||
|
|
|
||
|
|
Dump global and module bindings configurations''')
|
||
|
|
|
||
|
|
seiscomp.client.Application.printUsage(self)
|
||
|
|
|
||
|
|
print('''Examples:
|
||
|
|
Write bindings configuration from key directory to a configuration XML file:
|
||
|
|
bindings2cfg --key-dir ./etc/key -o config.xml
|
||
|
|
|
||
|
|
Write bindings configuration from key directory to the seiscomp local database
|
||
|
|
bindings2cfg --key-dir ./etc/key -d mysql://sysop:sysop@localhost/seiscomp
|
||
|
|
''')
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def send(self, *args):
|
||
|
|
'''
|
||
|
|
A simple wrapper that sends a message and tries to resend it in case of
|
||
|
|
an error.
|
||
|
|
'''
|
||
|
|
while not self.connection().send(*args):
|
||
|
|
sys.stderr.write("Warning: sending failed, retrying\n")
|
||
|
|
time.sleep(1)
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
'''
|
||
|
|
Reimplements the main loop of the application. This methods collects
|
||
|
|
all bindings and updates the database. It searches for already existing
|
||
|
|
objects and updates them or creates new objects. Objects that is didn't
|
||
|
|
touched are removed. This tool is the only one that should writes the
|
||
|
|
configuration into the database and thus manages the content.
|
||
|
|
'''
|
||
|
|
config = seiscomp.client.ConfigDB.Instance().config()
|
||
|
|
if config is None:
|
||
|
|
config = seiscomp.datamodel.Config()
|
||
|
|
|
||
|
|
configMod = None
|
||
|
|
obsoleteConfigMods = []
|
||
|
|
|
||
|
|
if not self._outputFile:
|
||
|
|
moduleName = self.name()
|
||
|
|
seiscomp.datamodel.Notifier.Enable()
|
||
|
|
else:
|
||
|
|
moduleName = "trunk"
|
||
|
|
|
||
|
|
configID = "Config/%s" % moduleName
|
||
|
|
|
||
|
|
for i in range(config.configModuleCount()):
|
||
|
|
if config.configModule(i).publicID() != configID:
|
||
|
|
obsoleteConfigMods.append(config.configModule(i))
|
||
|
|
else:
|
||
|
|
configMod = config.configModule(i)
|
||
|
|
|
||
|
|
# Remove obsolete config modules
|
||
|
|
for cm in obsoleteConfigMods:
|
||
|
|
sys.stderr.write(
|
||
|
|
"- %s / obsolete module configuration\n" % cm.name())
|
||
|
|
ps = seiscomp.datamodel.ParameterSet.Find(cm.parameterSetID())
|
||
|
|
if not ps is None:
|
||
|
|
ps.detach()
|
||
|
|
cm.detach()
|
||
|
|
del obsoleteConfigMods
|
||
|
|
|
||
|
|
if not configMod:
|
||
|
|
configMod = seiscomp.datamodel.ConfigModule.Find(configID)
|
||
|
|
if configMod is None:
|
||
|
|
configMod = seiscomp.datamodel.ConfigModule.Create(configID)
|
||
|
|
config.add(configMod)
|
||
|
|
else:
|
||
|
|
if configMod.name() != moduleName:
|
||
|
|
configMod.update()
|
||
|
|
if not configMod.enabled():
|
||
|
|
configMod.update()
|
||
|
|
|
||
|
|
configMod.setName(moduleName)
|
||
|
|
configMod.setEnabled(True)
|
||
|
|
else:
|
||
|
|
if configMod.name() != moduleName:
|
||
|
|
configMod.setName(moduleName)
|
||
|
|
configMod.update()
|
||
|
|
paramSet = seiscomp.datamodel.ParameterSet.Find(
|
||
|
|
configMod.parameterSetID())
|
||
|
|
if configMod.parameterSetID():
|
||
|
|
configMod.setParameterSetID("")
|
||
|
|
configMod.update()
|
||
|
|
|
||
|
|
if not paramSet is None:
|
||
|
|
paramSet.detach()
|
||
|
|
|
||
|
|
stationConfigs = {}
|
||
|
|
obsoleteStationConfigs = []
|
||
|
|
|
||
|
|
for i in range(configMod.configStationCount()):
|
||
|
|
cs = configMod.configStation(i)
|
||
|
|
if (cs.networkCode(), cs.stationCode()) in self.stationSetups:
|
||
|
|
stationConfigs[(cs.networkCode(), cs.stationCode())] = cs
|
||
|
|
else:
|
||
|
|
obsoleteStationConfigs.append(cs)
|
||
|
|
|
||
|
|
for cs in obsoleteStationConfigs:
|
||
|
|
sys.stderr.write("- %s/%s/%s / obsolete station configuration\n" %
|
||
|
|
(configMod.name(), cs.networkCode(), cs.stationCode()))
|
||
|
|
cs.detach()
|
||
|
|
del obsoleteStationConfigs
|
||
|
|
|
||
|
|
for staid, setups in list(self.stationSetups.items()):
|
||
|
|
try:
|
||
|
|
cs = stationConfigs[staid]
|
||
|
|
except:
|
||
|
|
cs = seiscomp.datamodel.ConfigStation.Find(
|
||
|
|
"Config/%s/%s/%s" % (configMod.name(), staid[0], staid[1]))
|
||
|
|
if not cs:
|
||
|
|
cs = seiscomp.datamodel.ConfigStation.Create(
|
||
|
|
"Config/%s/%s/%s" % (configMod.name(), staid[0], staid[1]))
|
||
|
|
configMod.add(cs)
|
||
|
|
cs.setNetworkCode(staid[0])
|
||
|
|
cs.setStationCode(staid[1])
|
||
|
|
cs.setEnabled(True)
|
||
|
|
|
||
|
|
ci = seiscomp.datamodel.CreationInfo()
|
||
|
|
ci.setCreationTime(seiscomp.core.Time.GMT())
|
||
|
|
ci.setAgencyID(self.agencyID())
|
||
|
|
ci.setAuthor(self.name())
|
||
|
|
cs.setCreationInfo(ci)
|
||
|
|
|
||
|
|
stationSetups = {}
|
||
|
|
obsoleteSetups = []
|
||
|
|
for i in range(cs.setupCount()):
|
||
|
|
setup = cs.setup(i)
|
||
|
|
if setup.name() in setups:
|
||
|
|
stationSetups[setup.name()] = setup
|
||
|
|
else:
|
||
|
|
obsoleteSetups.append(setup)
|
||
|
|
|
||
|
|
for s in obsoleteSetups:
|
||
|
|
sys.stderr.write("- %s/%s/%s/%s / obsolete station setup\n" %
|
||
|
|
(configMod.name(), cs.networkCode(), cs.stationCode(), setup.name()))
|
||
|
|
ps = seiscomp.datamodel.ParameterSet.Find(s.parameterSetID())
|
||
|
|
if ps:
|
||
|
|
ps.detach()
|
||
|
|
s.detach()
|
||
|
|
del obsoleteSetups
|
||
|
|
|
||
|
|
newParamSets = {}
|
||
|
|
globalSet = ""
|
||
|
|
for mod, params in list(setups.items()):
|
||
|
|
try:
|
||
|
|
setup = stationSetups[mod]
|
||
|
|
except:
|
||
|
|
setup = seiscomp.datamodel.Setup()
|
||
|
|
setup.setName(mod)
|
||
|
|
setup.setEnabled(True)
|
||
|
|
cs.add(setup)
|
||
|
|
|
||
|
|
paramSet = seiscomp.datamodel.ParameterSet.Find(
|
||
|
|
setup.parameterSetID())
|
||
|
|
if not paramSet:
|
||
|
|
paramSet = seiscomp.datamodel.ParameterSet.Find("ParameterSet/%s/Station/%s/%s/%s" % (
|
||
|
|
configMod.name(), cs.networkCode(), cs.stationCode(), setup.name()))
|
||
|
|
if not paramSet:
|
||
|
|
paramSet = seiscomp.datamodel.ParameterSet.Create(
|
||
|
|
"ParameterSet/%s/Station/%s/%s/%s" % (configMod.name(), cs.networkCode(), cs.stationCode(), setup.name()))
|
||
|
|
config.add(paramSet)
|
||
|
|
paramSet.setModuleID(configMod.publicID())
|
||
|
|
paramSet.setCreated(seiscomp.core.Time.GMT())
|
||
|
|
newParamSets[paramSet.publicID()] = 1
|
||
|
|
setup.setParameterSetID(paramSet.publicID())
|
||
|
|
if mod in stationSetups:
|
||
|
|
setup.update()
|
||
|
|
elif paramSet.moduleID() != configMod.publicID():
|
||
|
|
paramSet.setModuleID(configMod.publicID())
|
||
|
|
paramSet.update()
|
||
|
|
|
||
|
|
# Synchronize existing parameterset with the new parameters
|
||
|
|
sync(paramSet, params)
|
||
|
|
|
||
|
|
if setup.name() == "default":
|
||
|
|
globalSet = paramSet.publicID()
|
||
|
|
|
||
|
|
for i in range(cs.setupCount()):
|
||
|
|
setup = cs.setup(i)
|
||
|
|
paramSet = seiscomp.datamodel.ParameterSet.Find(
|
||
|
|
setup.parameterSetID())
|
||
|
|
if not paramSet:
|
||
|
|
continue
|
||
|
|
|
||
|
|
if paramSet.publicID() != globalSet and paramSet.baseID() != globalSet:
|
||
|
|
paramSet.setBaseID(globalSet)
|
||
|
|
if not paramSet.publicID() in newParamSets:
|
||
|
|
paramSet.update()
|
||
|
|
|
||
|
|
# Collect unused ParameterSets
|
||
|
|
usedSets = {}
|
||
|
|
for i in range(config.configModuleCount()):
|
||
|
|
configMod = config.configModule(i)
|
||
|
|
for j in range(configMod.configStationCount()):
|
||
|
|
cs = configMod.configStation(j)
|
||
|
|
for k in range(cs.setupCount()):
|
||
|
|
setup = cs.setup(k)
|
||
|
|
collect(usedSets, setup.parameterSetID())
|
||
|
|
|
||
|
|
# Delete unused ParameterSets
|
||
|
|
i = 0
|
||
|
|
while i < config.parameterSetCount():
|
||
|
|
paramSet = config.parameterSet(i)
|
||
|
|
if not paramSet.publicID() in usedSets:
|
||
|
|
sys.stderr.write("- %s / obsolete parameter set\n" %
|
||
|
|
paramSet.publicID())
|
||
|
|
paramSet.detach()
|
||
|
|
else:
|
||
|
|
i = i + 1
|
||
|
|
|
||
|
|
# Generate output file and exit if configured
|
||
|
|
if self._outputFile:
|
||
|
|
ar = seiscomp.io.XMLArchive()
|
||
|
|
if not ar.create(self._outputFile):
|
||
|
|
sys.stderr.write(
|
||
|
|
"Failed to created output file: %s" % self._outputFile)
|
||
|
|
return False
|
||
|
|
|
||
|
|
ar.setFormattedOutput(True)
|
||
|
|
ar.writeObject(config)
|
||
|
|
ar.close()
|
||
|
|
return True
|
||
|
|
|
||
|
|
ncount = seiscomp.datamodel.Notifier.Size()
|
||
|
|
if ncount > 0:
|
||
|
|
sys.stderr.write("+ synchronize %d change" % ncount)
|
||
|
|
if ncount > 1:
|
||
|
|
sys.stderr.write("s")
|
||
|
|
sys.stderr.write("\n")
|
||
|
|
else:
|
||
|
|
sys.stderr.write("- database is already up-to-date\n")
|
||
|
|
return True
|
||
|
|
|
||
|
|
cfgmsg = seiscomp.datamodel.ConfigSyncMessage(False)
|
||
|
|
cfgmsg.setCreationInfo(seiscomp.datamodel.CreationInfo())
|
||
|
|
cfgmsg.creationInfo().setCreationTime(seiscomp.core.Time.GMT())
|
||
|
|
cfgmsg.creationInfo().setAuthor(self.author())
|
||
|
|
cfgmsg.creationInfo().setAgencyID(self.agencyID())
|
||
|
|
self.send(seiscomp.client.Protocol.STATUS_GROUP, cfgmsg)
|
||
|
|
|
||
|
|
# Send messages in a batch of 100 notifiers to not exceed the
|
||
|
|
# maximum allowed message size of ~300kb.
|
||
|
|
msg = seiscomp.datamodel.NotifierMessage()
|
||
|
|
nmsg = seiscomp.datamodel.Notifier.GetMessage(False)
|
||
|
|
count = 0
|
||
|
|
sys.stderr.write("\r + sending notifiers: %d%%" %
|
||
|
|
(count * 100 / ncount))
|
||
|
|
sys.stderr.flush()
|
||
|
|
while nmsg:
|
||
|
|
for o in nmsg:
|
||
|
|
n = seiscomp.datamodel.Notifier.Cast(o)
|
||
|
|
if n:
|
||
|
|
msg.attach(n)
|
||
|
|
|
||
|
|
if msg.size() >= 100:
|
||
|
|
count += msg.size()
|
||
|
|
self.send("CONFIG", msg)
|
||
|
|
msg.clear()
|
||
|
|
sys.stderr.write("\r + sending notifiers: %d%%" %
|
||
|
|
(count * 100 / ncount))
|
||
|
|
sys.stderr.flush()
|
||
|
|
|
||
|
|
nmsg = seiscomp.datamodel.Notifier.GetMessage(False)
|
||
|
|
|
||
|
|
if msg.size() > 0:
|
||
|
|
count += msg.size()
|
||
|
|
self.send("CONFIG", msg)
|
||
|
|
msg.clear()
|
||
|
|
sys.stderr.write("\r + sending notifiers: %d%%" %
|
||
|
|
(count * 100 / ncount))
|
||
|
|
sys.stderr.flush()
|
||
|
|
|
||
|
|
sys.stderr.write("\n")
|
||
|
|
|
||
|
|
# Notify about end of synchronization
|
||
|
|
cfgmsg.creationInfo().setCreationTime(seiscomp.core.Time.GMT())
|
||
|
|
cfgmsg.isFinished = True
|
||
|
|
self.send(seiscomp.client.Protocol.STATUS_GROUP, cfgmsg)
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
app = ConfigDBUpdater(len(sys.argv), sys.argv)
|
||
|
|
return app()
|