You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
381 lines
14 KiB
Plaintext
381 lines
14 KiB
Plaintext
1 year ago
|
#!/usr/bin/env seiscomp-python
|
||
|
|
|
||
|
|
################################################################################
|
||
|
|
# Copyright (C) 2012-2013, 2020 Helmholtz-Zentrum Potsdam - Deutsches GeoForschungsZentrum GFZ
|
||
|
|
#
|
||
|
|
# tabinvmodifier -- Tool for inventory modification using nettab files.
|
||
|
|
#
|
||
|
|
# This software is free software and comes with ABSOLUTELY NO WARRANTY.
|
||
|
|
#
|
||
|
|
# Author: Marcelo Bianchi
|
||
|
|
# Email: mbianchi@gfz-potsdam.de
|
||
|
|
################################################################################
|
||
|
|
|
||
|
|
from __future__ import print_function
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import datetime, time
|
||
|
|
from nettab.lineType import Nw, Sa, Na, Ia
|
||
|
|
from nettab.basesc3 import sc3
|
||
|
|
import seiscomp.datamodel, seiscomp.io, seiscomp.client, seiscomp.core, seiscomp.logging
|
||
|
|
|
||
|
|
class Rules(object):
|
||
|
|
def __init__(self, relaxed = False):
|
||
|
|
self.relaxed = relaxed
|
||
|
|
self.attributes = {}
|
||
|
|
self.iattributes = []
|
||
|
|
return
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _overlaps(pstart, pend, cstart, cend):
|
||
|
|
if pend:
|
||
|
|
if pend > cstart:
|
||
|
|
if not cend or pstart < cend:
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
if not cend or pstart < cend:
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def Nw(self, nw):
|
||
|
|
key = (nw.code, nw.start, nw.end)
|
||
|
|
if key in self.attributes:
|
||
|
|
raise Exception("Nw (%s/%s-%s) is already defined." % key)
|
||
|
|
self.attributes[key] = {}
|
||
|
|
self.attributes[key]["Sa"] = []
|
||
|
|
self.attributes[key]["Na"] = []
|
||
|
|
return key
|
||
|
|
|
||
|
|
def Sa(self, key, sa):
|
||
|
|
try:
|
||
|
|
items = self.attributes[key]["Sa"]
|
||
|
|
except KeyError:
|
||
|
|
raise Exception ("Nw %s/%s-%s not found in Ruleset" % key)
|
||
|
|
items.append(sa)
|
||
|
|
|
||
|
|
def Na(self, key, na):
|
||
|
|
try:
|
||
|
|
items = self.attributes[key]["Na"]
|
||
|
|
except KeyError:
|
||
|
|
raise Exception ("Nw %s/%s-%s not found in Ruleset" % key)
|
||
|
|
items.append(na)
|
||
|
|
|
||
|
|
def Ia(self, ia):
|
||
|
|
self.iattributes.append(ia);
|
||
|
|
|
||
|
|
def findKey(self, ncode, nstart, nend):
|
||
|
|
for (code, start, end) in self.attributes:
|
||
|
|
if code == ncode and self._overlaps(start, end, nstart, nend):
|
||
|
|
return (code, start, end)
|
||
|
|
return None
|
||
|
|
|
||
|
|
def getInstrumentsAttributes(self, elementId, elementType):
|
||
|
|
att = {}
|
||
|
|
for item in self.iattributes:
|
||
|
|
if item.match(elementId, elementType):
|
||
|
|
att[item.Key] = item.Value
|
||
|
|
return att
|
||
|
|
|
||
|
|
def getNetworkAttributes(self, key):
|
||
|
|
att = {}
|
||
|
|
for item in self.attributes[key]["Na"]:
|
||
|
|
att[item.Key] = item.Value
|
||
|
|
return att
|
||
|
|
|
||
|
|
def getStationAttributes(self, key, ncode, scode, lcode, ccode, start, end):
|
||
|
|
att = {}
|
||
|
|
for item in self.attributes[key]["Sa"]:
|
||
|
|
if item.match(scode, lcode, ccode, start, end, self.relaxed):
|
||
|
|
att[item.Key] = item.Value
|
||
|
|
return att
|
||
|
|
|
||
|
|
class InventoryModifier(seiscomp.client.Application):
|
||
|
|
def __init__(self, argc, argv):
|
||
|
|
seiscomp.client.Application.__init__(self, argc, argv)
|
||
|
|
self.setMessagingUsername("iModify")
|
||
|
|
|
||
|
|
self.rules = None
|
||
|
|
self.relaxed = False
|
||
|
|
self.outputFile = None
|
||
|
|
|
||
|
|
def _digest(self, tabFilename, rules = None):
|
||
|
|
if not tabFilename or not os.path.isfile(tabFilename):
|
||
|
|
raise Exception("Supplied filename is invalid.")
|
||
|
|
|
||
|
|
if not rules:
|
||
|
|
rules = Rules(self.relaxed)
|
||
|
|
|
||
|
|
try:
|
||
|
|
fd = open(tabFilename)
|
||
|
|
for line in fd:
|
||
|
|
obj = None
|
||
|
|
line = line.strip()
|
||
|
|
if not line or line[0] == "#": continue
|
||
|
|
if str(line).find(":") == -1:
|
||
|
|
raise Exception("Invalid line format '%s'" % line)
|
||
|
|
(Type, Content) = line.split(":",1)
|
||
|
|
|
||
|
|
if Type == "Nw":
|
||
|
|
nw = Nw(Content)
|
||
|
|
key = rules.Nw(nw)
|
||
|
|
elif Type == "Sg":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
elif Type == "Na":
|
||
|
|
na = Na(Content)
|
||
|
|
rules.Na(key, na)
|
||
|
|
elif Type == "Sa":
|
||
|
|
sa = Sa(Content)
|
||
|
|
rules.Sa(key, sa)
|
||
|
|
elif Type == "Sr":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
elif Type == "Ia":
|
||
|
|
ia = Ia(Content)
|
||
|
|
rules.Ia(ia)
|
||
|
|
elif Type == "Se":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
elif Type == "Dl":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
elif Type == "Cl":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
elif Type == "Ff":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
elif Type == "If":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
elif Type == "Pz":
|
||
|
|
raise Exception("Type not supported.")
|
||
|
|
except Exception as e:
|
||
|
|
raise e
|
||
|
|
|
||
|
|
finally:
|
||
|
|
if fd:
|
||
|
|
fd.close()
|
||
|
|
return rules
|
||
|
|
|
||
|
|
def validateParameters(self):
|
||
|
|
outputFile = None
|
||
|
|
rulesFile = None
|
||
|
|
|
||
|
|
if self.commandline().hasOption("rules"):
|
||
|
|
rulesFile = self.commandline().optionString("rules")
|
||
|
|
|
||
|
|
if self.commandline().hasOption("output"):
|
||
|
|
outputFile = self.commandline().optionString("output")
|
||
|
|
|
||
|
|
if self.commandline().hasOption("relaxed"):
|
||
|
|
self.relaxed = True
|
||
|
|
|
||
|
|
if self.commandline().hasOption("inventory-db") and outputFile is None:
|
||
|
|
print("Cannot send notifiers when loading inventory from file.", file=sys.stderr)
|
||
|
|
return False
|
||
|
|
|
||
|
|
if self.commandline().unrecognizedOptions():
|
||
|
|
print("Invalid options: ", end=' ', file=sys.stderr)
|
||
|
|
for i in self.commandline().unrecognizedOptions():
|
||
|
|
print(i, end=' ', file=sys.stderr)
|
||
|
|
print("", file=sys.stderr)
|
||
|
|
return False
|
||
|
|
|
||
|
|
if not rulesFile:
|
||
|
|
print("No rule file was supplied for processing", file=sys.stderr)
|
||
|
|
return False
|
||
|
|
|
||
|
|
if not os.path.isfile(rulesFile):
|
||
|
|
argv0 = os.path.basename(self.arguments()[0])
|
||
|
|
print("%s: %s: No such file or directory" % (argv0, rulesFile), file=sys.stderr)
|
||
|
|
return False
|
||
|
|
|
||
|
|
if self.commandline().hasOption("inventory-db"):
|
||
|
|
self.setDatabaseEnabled(False, False)
|
||
|
|
self.setMessagingEnabled(False)
|
||
|
|
|
||
|
|
self.rules = self._digest(rulesFile, self.rules)
|
||
|
|
self.outputFile = outputFile
|
||
|
|
return True
|
||
|
|
|
||
|
|
def createCommandLineDescription(self):
|
||
|
|
seiscomp.client.Application.createCommandLineDescription(self)
|
||
|
|
|
||
|
|
self.commandline().addGroup("Rules")
|
||
|
|
self.commandline().addStringOption("Rules", "rules,r", "Input XML filename")
|
||
|
|
self.commandline().addOption("Rules", "relaxed,e", "Relax rules for matching NSLC items")
|
||
|
|
|
||
|
|
self.commandline().addGroup("Dump")
|
||
|
|
self.commandline().addStringOption("Dump", "output,o", "Output XML filename")
|
||
|
|
|
||
|
|
def initConfiguration(self):
|
||
|
|
value = seiscomp.client.Application.initConfiguration(self)
|
||
|
|
self.setLoggingToStdErr(True)
|
||
|
|
self.setDatabaseEnabled(True, True)
|
||
|
|
self.setMessagingEnabled(True)
|
||
|
|
self.setLoadInventoryEnabled(True)
|
||
|
|
return value
|
||
|
|
|
||
|
|
def send(self, *args):
|
||
|
|
while not self.connection().send(*args):
|
||
|
|
seiscomp.logging.warning("send failed, retrying")
|
||
|
|
time.sleep(1)
|
||
|
|
|
||
|
|
def send_notifiers(self, group):
|
||
|
|
Nsize = seiscomp.datamodel.Notifier.Size()
|
||
|
|
|
||
|
|
if Nsize > 0:
|
||
|
|
seiscomp.logging.info("trying to apply %d change%s" % (Nsize,"s" if Nsize != 1 else "" ))
|
||
|
|
else:
|
||
|
|
seiscomp.logging.info("no changes to apply")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
Nmsg = seiscomp.datamodel.Notifier.GetMessage(True)
|
||
|
|
it = Nmsg.iter()
|
||
|
|
msg = seiscomp.datamodel.NotifierMessage()
|
||
|
|
|
||
|
|
maxmsg = 100
|
||
|
|
sent = 0
|
||
|
|
mcount = 0
|
||
|
|
|
||
|
|
try:
|
||
|
|
try:
|
||
|
|
while it.get():
|
||
|
|
msg.attach(seiscomp.datamodel.Notifier_Cast(it.get()))
|
||
|
|
mcount += 1
|
||
|
|
if msg and mcount == maxmsg:
|
||
|
|
sent += mcount
|
||
|
|
seiscomp.logging.debug("sending message (%5.1f %%)" % (sent / float(Nsize) * 100.0))
|
||
|
|
self.send(group, msg)
|
||
|
|
msg.clear()
|
||
|
|
mcount = 0
|
||
|
|
next(it)
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
finally:
|
||
|
|
if msg.size():
|
||
|
|
seiscomp.logging.debug("sending message (%5.1f %%)" % 100.0)
|
||
|
|
self.send(group, msg)
|
||
|
|
msg.clear()
|
||
|
|
seiscomp.logging.info("done")
|
||
|
|
return mcount
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _loop(obj, count):
|
||
|
|
return [ obj(i) for i in range(count) ]
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _collect(obj):
|
||
|
|
code = obj.code()
|
||
|
|
start = datetime.datetime.strptime(obj.start().toString("%Y %m %d %H %M %S"), "%Y %m %d %H %M %S")
|
||
|
|
try:
|
||
|
|
end = obj.end()
|
||
|
|
end = datetime.datetime.strptime(end.toString("%Y %m %d %H %M %S"), "%Y %m %d %H %M %S")
|
||
|
|
except:
|
||
|
|
end = None
|
||
|
|
return (code, start, end)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _modifyInventory(mode, obj, att):
|
||
|
|
valid = sc3._findValidOnes(mode)
|
||
|
|
if not att:
|
||
|
|
return
|
||
|
|
|
||
|
|
# Why repeat the code in basesc3.py (sc3::_fillSc3())?
|
||
|
|
# What about if there are existing comments/pids - won't
|
||
|
|
# this code get the count wrong?? *FIXME*
|
||
|
|
commentNum = 0
|
||
|
|
for (k,p) in att.items():
|
||
|
|
try:
|
||
|
|
if k == 'Comment':
|
||
|
|
# print('DEBUG: Adding comment', p)
|
||
|
|
if p.startswith('Grant'):
|
||
|
|
# 2020: These belong in DOI metadata, not here.
|
||
|
|
continue
|
||
|
|
|
||
|
|
c = seiscomp.datamodel.Comment()
|
||
|
|
c.setText(p)
|
||
|
|
c.setId(str(commentNum))
|
||
|
|
commentNum += 1
|
||
|
|
obj.add(c)
|
||
|
|
continue
|
||
|
|
|
||
|
|
if k == 'Pid':
|
||
|
|
print('DEBUG: Adding Pid as comment', p)
|
||
|
|
c = seiscomp.datamodel.Comment()
|
||
|
|
(typ, val) = p.split(':', 1)
|
||
|
|
s = '{"type":"%s", "value":"%s"}' % (typ.upper(), val)
|
||
|
|
c.setText(s)
|
||
|
|
c.setId('FDSNXML:Identifier/' + str(commentNum))
|
||
|
|
commentNum += 1
|
||
|
|
obj.add(c)
|
||
|
|
continue
|
||
|
|
|
||
|
|
p = valid['attributes'][k]['validator'](p)
|
||
|
|
getattr(obj, 'set'+k)(p)
|
||
|
|
except KeyError:
|
||
|
|
import string
|
||
|
|
hint = ''
|
||
|
|
if k[0] in string.lowercase:
|
||
|
|
hint = " (try '%s' instead)" % ( k[0].upper() + k[1:])
|
||
|
|
print('Modifying %s: \'%s\' is not a valid key%s' % (mode, k, hint), file=sys.stderr)
|
||
|
|
obj.update()
|
||
|
|
return
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
rules = self.rules
|
||
|
|
iv = seiscomp.client.Inventory.Instance().inventory()
|
||
|
|
|
||
|
|
if not rules:
|
||
|
|
return False
|
||
|
|
|
||
|
|
if not iv:
|
||
|
|
return False
|
||
|
|
|
||
|
|
seiscomp.logging.debug("Loaded %d networks" % iv.networkCount())
|
||
|
|
if self.outputFile is None:
|
||
|
|
seiscomp.datamodel.Notifier.Enable()
|
||
|
|
self.setInterpretNotifierEnabled(True)
|
||
|
|
|
||
|
|
for net in self._loop(iv.network, iv.networkCount()):
|
||
|
|
(ncode, nstart, nend) = self._collect(net)
|
||
|
|
key = rules.findKey(ncode, nstart, nend)
|
||
|
|
if not key: continue
|
||
|
|
att = rules.getNetworkAttributes(key)
|
||
|
|
self._modifyInventory("network", net, att)
|
||
|
|
seiscomp.logging.info("%s %s" % (ncode, att))
|
||
|
|
for sta in self._loop(net.station, net.stationCount()):
|
||
|
|
(scode, sstart, send) = self._collect(sta)
|
||
|
|
att = rules.getStationAttributes(key, ncode, scode, None, None, sstart, send)
|
||
|
|
self._modifyInventory("station", sta, att)
|
||
|
|
if att: seiscomp.logging.info(" %s %s" % (scode, att))
|
||
|
|
for loc in self._loop(sta.sensorLocation, sta.sensorLocationCount()):
|
||
|
|
(lcode, lstart, lend) = self._collect(loc)
|
||
|
|
att = rules.getStationAttributes(key, ncode, scode, lcode, None, lstart, lend)
|
||
|
|
self._modifyInventory("location", loc, att)
|
||
|
|
if att: seiscomp.logging.info(" %s %s" % (lcode, att))
|
||
|
|
for cha in self._loop(loc.stream, loc.streamCount()):
|
||
|
|
(ccode, cstart, cend) = self._collect(cha)
|
||
|
|
att = rules.getStationAttributes(key, ncode, scode, lcode, ccode, cstart, cend)
|
||
|
|
self._modifyInventory("channel", cha, att)
|
||
|
|
if att: seiscomp.logging.info(" %s %s" % (ccode, att))
|
||
|
|
|
||
|
|
for sensor in self._loop(iv.sensor, iv.sensorCount()):
|
||
|
|
att = rules.getInstrumentsAttributes(sensor.name(), "Se")
|
||
|
|
self._modifyInventory("sensor", sensor, att)
|
||
|
|
|
||
|
|
for datalogger in self._loop(iv.datalogger, iv.dataloggerCount()):
|
||
|
|
att = rules.getInstrumentsAttributes(datalogger.name(), "Dl")
|
||
|
|
self._modifyInventory("datalogger", datalogger, att)
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def done(self):
|
||
|
|
if self.outputFile:
|
||
|
|
ar = seiscomp.io.XMLArchive()
|
||
|
|
ar.create(self.outputFile)
|
||
|
|
ar.setFormattedOutput(True)
|
||
|
|
ar.writeObject(seiscomp.client.Inventory.Instance().inventory())
|
||
|
|
ar.close()
|
||
|
|
else:
|
||
|
|
self.send_notifiers("INVENTORY")
|
||
|
|
seiscomp.client.Application.done(self)
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
app = InventoryModifier(len(sys.argv), sys.argv)
|
||
|
|
sys.exit(app())
|