SCRUG-NA/2025
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[seiscomp, scanloc] Install, add .gitignore

Browse Source
This commit is contained in:
Dirk Rössler
2025-10-09 15:07:02 +02:00
commit 20f5301bb1
2848 changed files with 1315858 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

787
bin/gempa-checkSCconfig Executable file
View File

@ -0,0 +1,787 @@
#!/usr/bin/env seiscomp-python
############################################################################
# Copyright (C) 2021 by gempa GmbH #
# #
# All Rights Reserved. #
# #
# NOTICE: All information contained herein is, and remains #
# the property of gempa GmbH and its suppliers, if any. The intellectual #
# and technical concepts contained herein are proprietary to gempa GmbH #
# and its suppliers. #
# Dissemination of this information or reproduction of this material #
# is strictly forbidden unless prior written permission is obtained #
# from gempa GmbH. #
# #
# Author: Dirk Roessler #
# Email: roessler@gempa.de #
############################################################################
from __future__ import absolute_import, division, print_function
import fnmatch
import sys
import os
import seiscomp.client
import seiscomp.io
import seiscomp.datamodel
import seiscomp.logging as logging
# define the latest version for which this script applies
latestVersion = "6.6.0"
# define module and binding configuration parameters
# define module and binding configuration parameter values
###############################################################################
# binding parameters
parameters = {
###########################################################################
# bindigng parameter names
# magnitudes
# MLh
"MLh.maxavg, version:4.5.0": "amplitudes.MLh.params",
"MLh.ClippingThreshold, version:4.5.0": "amplitudes.MLh.ClippingThreshold",
"MLh.params, version:4.5.0": "magnitudes.MLh.params",
# md
"md.maxavg, version:4.5.0": "magnitudes.md.seismo",
"md.taper, version:4.5.0": "magnitudes.md.taper",
"md.signal_length, version:4.5.0": "magnitudes.md.signal_length",
"md.butterworth, version:4.5.0": "magnitudes.md.butterworth",
"md.depthmax, version:4.5.0": "magnitudes.md.depthmax",
"md.deltamax, version:4.5.0": "magnitudes.md.deltamax",
"md.snrmin, version:4.5.0": "magnitudes.md.snrmin",
"md.mdmax, version:4.5.0": "magnitudes.md.mdmax",
"md.fma, version:4.5.0": "magnitudes.md.fma",
"md.fmb, version:4.5.0": "magnitudes.md.fmb",
"md.fmd, version:4.5.0": "magnitudes.md.fmd",
"md.fmf, version:4.5.0": "magnitudes.md.fmf",
"md.fmz, version:4.5.0": "magnitudes.md.fmz",
"md.linearcorrection, version:4.5.0": "magnitudes.md.linearcorrection",
"md.offset, version:4.5.0": "magnitudes.md.offset",
"md.stacor, version:4.5.0": "magnitudes.md.stacor",
# MLr
"MLr.maxavg, version:4.5.0": "magnitudes.MLr.params",
# Ms_20
"Ms_20.lowerPeriod, version:4.5.0": "magnitudes.Ms_20.lowerPeriod",
"Ms_20.upperPeriod, version:4.5.0": "magnitudes.Ms_20.upperPeriod",
"Ms_20.minDist, version:4.5.0": "magnitudes.Ms_20.minDist",
"Ms_20.maxDist, version:4.5.0": "magnitudes.Ms_20.maxDist",
"Ms_20.maxDepth, version:4.5.0": "magnitudes.Ms_20.maxDepth",
# MLv
"MLv.logA0, version:4.5.0": "magnitudes.MLv.logA0",
"MLv.maxDistanceKm, version:4.5.0": "magnitudes.MLv.maxDistanceKm",
# ML
"ML.logA0, version:4.5.0": "magnitudes.ML.logA0",
"ML.maxDistanceKm, version:4.5.0": "magnitudes.ML.maxDistanceKm",
###########################################################################
###########################################################################
# module parameter names
# scmv
"legend, version:4.6.0": "scheme.map.showLegends",
# data base
"database.type, version:5.0.0": "database",
"database.parameters, version:5.0.0": "database",
# RecordsStream
"recordstream.service, version:5.0.0": "recordstream",
"recordstream.source, version:5.0.0": "recordstream",
# scautoloc
"autoloc.locator.profile, version:4.3.0": "locator.profile",
"autoloc.cleanupInterval, version:4.3.0": "buffer.cleanupInterval",
"autoloc.maxAge, version:4.3.0": "buffer.pickKeep",
"autoloc.wakeupInterval, version:4.3.0": "",
# magnitudes
"magnitudes.*.regions, version:5.0.0": "magnitudes.*.regionFile",
###########################################################################
# SC 5.0.0
# global parameters for eventlist
"eventlist.customColumn, version:5.0.0": "eventlist.customColumn.name",
"eventlist.regions, version:5.0.0": "eventlist.filter.regions.profiles",
"eventlist.region, version:5.0.0": "eventlist.filter.regions.region",
# global parameters for eventedit
"eventedit.customColumn, version:5.0.0": "eventedit.origin.customColumn.name",
"eventedit.customColumn.default, version:5.0.0": "eventedit.origin.customColumn.default",
"eventedit.customColumn.originCommentID, version:5.0.0": "eventedit.origin.customColumn.originCommentID",
"eventedit.customColumn.pos, version:5.0.0": "eventedit.origin.customColumn.pos",
"eventedit.customColumn.colors, version:5.0.0": "eventedit.origin.customColumn.colors",
###########################################################################
# SC 5.1.1
# scolv
"picker.auxilliary., version:5.1.1": "picker.auxiliary.",
# SC 5.4.0
# ttt.homogeneous
"ttt.homogeneous.profile., version:5.4.0": "ttt.homogeneous.",
# SC 6.0.0
# scardac
"batchSize, version:6.0.0": "",
# SC 6.5.0
# StdLoc
"GridSearch.cellSize, version:6.5.0": "GridSearch.numPoints",
# scanloc
"clusterSearch.ignorePicks, version:scanloc": "buffer.ignorePickTimeDifference",
"clusterSearch.ignorePickTimeDifference, version:scanloc": "buffer.ignorePickTimeDifference",
"buffer.originAuthorWhiteList, version:scanloc": "buffer.authorWhiteList",
"score.weight.p, version:scanloc": "score.sum.weight.p",
"score.weight.s, version:scanloc": "score.sum.weight.s",
"score.weight.p0, version:scanloc": "score.sum.weight.p0",
"score.weight.s0, version:scanloc": "score.sum.weight.s0",
"score.weight.residual, version:scanloc": "score.sum.weight.residual",
"score.weight.depth, version:scanloc": "score.sum.weight.depth",
"score.weight.increaseManual, version:scanloc": "score.sum.weight.increaseManual",
"ossum.p, version:scanloc": "score.sum.weight.p",
"ossum.s, version:scanloc": "score.sum.weight.s",
"ossum.p0, version:scanloc": "score.sum.weight.p0",
"ossum.s0, version:scanloc": "score.sum.weight.s0",
"ossum.residual, version:scanloc": "score.sum.weight.residual",
"ossum.depth, version:scanloc": "score.sum.weight.depth",
"ossum.increaseManual, version:scanloc": "score.sum.weight.increaseManual",
"ossum.normalizationDepth, version:scanloc": "score.sum.weight.normalizationDepth",
"ossum.normalizationRMS, version:scanloc": "score.sum.weight.normalizationRMS",
"clustersearch.streamCheckLevel, version:scanloc": "streamCheckLevel",
# vortex
"script.*.path, version:vortex": "event.script.*.path",
# TOAST
"alwaysShowAdditionalOptions, version:toast": "dissemination.alwaysShowAdditionalOptions",
"confirmationRequired, version:toast": "dissemination.confirmationRequired",
"disseminationSelectionBehavior, version:toast": "dissemination.selectionBehavior",
}
###############################################################################
# parameter values
# module and binding configuration
valuesCfg = {
# plugins
"parameter:plugins,rsas, version:4.5.0": "",
# messaging
"parameter:plugins,dbplugin, version:4.0.0": "",
# events list
"parameter:eventlist.visibleColumns, TP, version:4.0.4": "MType for \
eventlist.visibleColumns",
# magnitude calibration functions
"parameter:logA0,0-1., version:5.0.0": "New format for logA0 parameter of \
magnitude: delta1:M1,delta2:M2,...",
# sigma
"parameter:plugins,gmpeeewd, version:sigma": "Replace 'gmpeeewd' by 'cppgmpe'",
"parameter:plugins,py3gmpe, version:sigma": "Replace 'py3gmpe' by 'cppgmpe' unless \
more Python3 GMPEs are needed",
"parameter:gempa.plugins,py3gmpe, version:sigma": "Remove and apply defaults or \
replace 'py3gmpe' by 'cppgmpe' unless more Python3 GMPEs are needed. \
Check cppgmpe/GMPE names!",
"parameter:gempa.plugins,pygmpe, version:sigma": "Remove and apply defaults or \
replace 'pygmpe' by 'cppgmpe' unless more Python GMPEs are needed. \
Check cppgmpe/GMPE names!",
"parameter:plugins,pygmpe, version:sigma": "Replace 'pygmpe' by 'cppgmpe' unless \
more Python GMPEs are needed",
"gfz-potsdam.de, version:4.0.0": "Replace by gfz.de - server address has changed",
}
###############################################################################
def gempaStatement(gempa):
print(
f""" + This parameter seems to belong to the gempa module '{gempa}'. The \
proposed action is valid
for the most recent version. Read the changelog of '{gempa}' before applying \
the changes!""",
file=sys.stderr,
)
return True
def checkParameter(inFile, oldValue, newValue):
found = 0
line = ""
with open(inFile, encoding="utf-8") as f:
lines = f.readlines()
lineNo = 0
for line in lines:
found = None
lineNo += 1
if line.startswith("#"):
continue
words = line.split()
# Check each possible substring
for i in range(len(words)):
for j in range(i + 1, len(words) + 1):
substring = " ".join(words[i:j])
if fnmatch.fnmatch(substring, oldValue) or oldValue in substring:
# return True
# if oldValue in line:
# try reducing false positives
idxStart = line.find(oldValue)
if idxStart != 0:
if "." == line[idxStart] or "," == line[idxStart]:
continue
idxEnd = idxStart + len(oldValue)
if idxEnd != len(oldValue) - 1:
if "." == line[idxEnd] or "," == line[idxEnd]:
continue
if newValue not in oldValue and newValue not in line:
found = lineNo
if newValue in oldValue:
found = lineNo
if found:
# print(
# f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated \
# parameter {oldValue} - new parameter: {newValue}",
# file=sys.stderr,
# )
# print(f" + full line: {line.rstrip()}", file=sys.stderr)
return found, line
return found, line
###############################################################################
def findValue(inFile, oldValue, parameter=None):
found = 0
line = ""
with open(inFile, encoding="utf-8") as f:
lines = f.readlines()
lineNo = 0
for line in lines:
lineNo += 1
if line.startswith("#"):
continue
if parameter and parameter not in line:
continue
if oldValue in line:
found = lineNo
return found, line
return found, line
###############################################################################
def printFinal(version, issuesFound=None):
print(
f"This check applies to SeisComP in version <= {version}",
file=sys.stderr,
)
print(" + read your own version, e.g.: 'seiscomp exec scm -V'", file=sys.stderr)
if issuesFound:
print(f" + found issues: {issuesFound}", file=sys.stderr)
else:
print(" + no issues found \U0001f44d", file=sys.stderr)
print(
"""
Alert and recommendation:
* Applies to: SeisComP databases using the charset utf8mb4 created with SeisComP in
version 6.0.0 <= version < 6.7.0 or nightly after 17 August 2023 until February 2025
* Issue: The charset used for the database does not discriminate upper and lower case
characters.
* Actions:
* Install the script 'gempa-check-database' with the package 'seiscomp-tools'
using gsm or download from https://data.gempa.de/packages/Public/.
* Stop scmaster
* Ensure, no other modules like scdb, scardac, etc. or custom tools from internal or
external clients attempt accessing the database.
* Login to your database, e.g.:
mysql -u sysop -p
* Source one of the suggested update scripts:
SOURCE /tmp/update-mysql-charset-CHARACTERSET-DATABASE.sql"""
)
return True
class CheckConfig(seiscomp.client.Application):
def __init__(self, argc, argv):
seiscomp.client.Application.__init__(self, argc, argv)
self.setMessagingEnabled(False)
self.setDatabaseEnabled(False, False)
self._configDir = False
self._home = os.environ["HOME"]
self._root = None
self._seiscompVersion = latestVersion
def createCommandLineDescription(self):
self.commandline().addGroup("SeisComP")
self.commandline().addStringOption(
"SeisComP",
"config-dir,c",
"Path to non-standard @CONFIGDIR@. Default: False",
)
self.commandline().addStringOption(
"SeisComP",
"root,r",
"SeisComP root directory to search for "
"SYSTEMCONFIGDIR. Default: $SEISCOMP_ROOT",
)
self.commandline().addStringOption(
"SeisComP",
"seiscomp-version, s",
"SeisComP version number to be considered for testing.",
)
def printUsage(self):
print(
f"""Usage:
{os.path.basename(__file__)} [options]
Identify + report legacy configuration for SeisComP and gempa modules in version <= \
{latestVersion}"""
)
seiscomp.client.Application.printUsage(self)
print(
f"""Examples:
For the test results read the log file or use --console 1
Simple run considering all parameters up version {self._seiscompVersion}
{os.path.basename(__file__)}
Run with specfic $SEISCOMP_ROOT directory and SeisComP version
{os.path.basename(__file__)} -r /home/sysop/seisomp-test --console 1 \
--seiscomp-version 4.8.0"""
)
def init(self):
if not seiscomp.client.Application.init(self):
return False
try:
self._root = self.commandline().optionString("root")
except RuntimeError:
try:
self._root = os.environ["SEISCOMP_ROOT"]
except KeyError:
print(
"SEISCOMP_ROOT directory is undefined. Cannot continue.",
file=sys.stderr,
)
return False
logging.debug(f"ROOT directory not given. Assuming {self._root}")
try:
self._configDir = self.commandline().optionString("config-dir")
except RuntimeError:
logging.debug(
f"Configuration directory not set. Creating from ROOT: {self._root}"
)
try:
self._seiscompVersion = self.commandline().optionString("seiscomp-version")
except RuntimeError:
logging.debug(f"SeisComP not set. Assuming version {self._seiscompVersion}")
return True
def run(self):
issuesFound = 0
if not os.path.exists(self._home):
print(
f"{self._home} does not exist, check your option --home",
file=sys.stderr,
)
return False
# define @CONFIGDIR@ / @SYSTEMCONFIGDIR@
if not self._configDir:
configSC = os.path.join(self._home, ".seiscomp")
configSC3 = os.path.join(self._home, ".seiscomp3")
systemConfigSC = os.path.join(self._root, "etc")
keysSC = os.path.join(self._root, "etc/key")
else:
configSC = self._configDir
configSC3 = configSC
systemConfigSC = os.path.join(configSC, "etc")
keysSC = os.path.join(configSC, "etc/key")
print("\nTesting general issues:", file=sys.stderr)
if os.path.exists(configSC3):
for configFile in os.listdir(configSC3):
if configFile.endswith("cfg"):
print(
"SeisComP3 configuration still exists in "
f"{configSC3} - consider migrating or removing",
file=sys.stderr,
)
issuesFound += 1
break
if not os.path.exists(configSC):
logging.debug(
f"SeisComP configuration '{configSC}' does not exist in '{self._home}'"
"and will not be tested. You may consider using '-c'."
)
return False
if not os.path.exists(systemConfigSC):
logging.error(
f"SeisComP system configuration {systemConfigSC} does not exist in "
f"{self._root}. Set your SeisComP variables or consider using '-r'."
)
return False
if not os.path.exists(keysSC):
logging.error(
f"SeisComP key directory {keysSC} does not exist in {self._root}. "
"Set your SeisComP variables or consider using '-r'"
)
return False
# test if old license path exists
oldLicenses = os.path.join(configSC, "licenses")
if os.path.exists(oldLicenses):
print(
f" + ISSUE: Found old license directory '{oldLicenses}'. Move it to "
"@DATADIR@/licenses",
file=sys.stderr,
)
print(
" + more information: "
"'https://www.gempa.de/news/2022/02/licenses/'",
file=sys.stderr,
)
issuesFound += 1
# filter parameters by considered version
parametersFiltered = {}
for test, newValue in parameters.items():
items = test.split(",")
oldValue = None
version = None
parameter = None
gempa = False
for item in items:
if "version" in item:
version = item.split(":")[1]
if len(version.split(".")) != 3:
gempa = version
continue
oldValue = item
if oldValue is None:
continue
if not gempa and version and version > self._seiscompVersion:
continue
parametersFiltered[oldValue] = [newValue, parameter, gempa]
# if ", version:" in oldValue:
# key = oldValue.split(", version:")[0]
# version = oldValue.split(", version:")[1]
# if version <= self._seiscompVersion:
# parametersFiltered[key] = newValue
# else:
# parametersFiltered[oldValue] = newValue
# module configuration parameter values
valuesCfgFiltered = {}
for oldValue, newValue in valuesCfg.items():
items = oldValue.split(",")
key = None
version = None
parameter = None
gempa = False
for item in items:
if "version" in item:
version = item.split(":")[1]
if len(version.split(".")) != 3:
gempa = version
elif "parameter" in item:
parameter = item.split(":")[1]
else:
key = item
if key is None:
continue
if not gempa and version and version > self._seiscompVersion:
continue
valuesCfgFiltered[key] = [newValue, parameter, gempa]
print("\nTesting module configurations in @CONFIGDIR@:", file=sys.stderr)
# test module configurations in CONFIGDIR
for config in os.listdir(configSC):
configFile = os.path.join(configSC, config)
if not os.path.isfile(configFile):
continue
if not configFile.endswith("cfg"):
continue
logging.debug(f" + testing module configurations in {configFile}")
print(f" + file {configFile}", file=sys.stderr)
# test parameters
for oldValue, parameter in parametersFiltered.items():
newValue = parameter[0]
gempa = parameter[2]
result = checkParameter(configFile, oldValue, newValue)
lineNo = result[0]
line = result[1]
if not lineNo:
continue
issuesFound += 1
print(
f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated "
f"parameter '{oldValue}'",
file=sys.stderr,
)
print(f" + full line: {line.rstrip()}", file=sys.stderr)
if newValue:
print(
f" + new parameter: '{newValue}'",
file=sys.stderr,
)
else:
print(
" + action: Remove the value",
file=sys.stderr,
)
if gempa:
gempaStatement(gempa)
# test values
for oldValue, new in valuesCfgFiltered.items():
newValue = new[0]
parameter = new[1]
gempa = new[2]
result = findValue(configFile, oldValue)
lineNo = result[0]
line = result[1]
if not lineNo:
continue
issuesFound += 1
if newValue:
print(
f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated "
f"parameter '{oldValue}' - new parameter: '{newValue}'\n"
" + action: Rename the parameter unless correct",
file=sys.stderr,
)
print(f" + full line: {line.rstrip()}", file=sys.stderr)
continue
print(
f" + POTENIAL ISSUE on line lineNo: obsolete/deprecated "
f"parameter '{oldValue}'\n"
" + action: Remove the parameter unless correct",
file=sys.stderr,
)
print(f" + full line: {line.rstrip()}", file=sys.stderr)
if gempa:
gempaStatement(gempa)
if issuesFound == 0:
print(" + found no issue", file=sys.stderr)
# test module configurations in SYSTEMCONFIGDIR
print(f"\nTesting module configurations in {systemConfigSC}:", file=sys.stderr)
for config in os.listdir(systemConfigSC):
configFile = os.path.join(systemConfigSC, config)
if not os.path.isfile(configFile):
continue
if not configFile.endswith("cfg"):
continue
logging.debug(f"testing module configurations in {configFile}")
print(f" + file {configFile}", file=sys.stderr)
# test parameters
for oldValue, parameter in parametersFiltered.items():
newValue = parameter[0]
gempa = parameter[2]
result = checkParameter(configFile, oldValue, newValue)
lineNo = result[0]
line = result[1]
if not lineNo:
continue
issuesFound += 1
print(
f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated "
f"parameter '{oldValue}'",
file=sys.stderr,
)
print(f" + full line: {line.rstrip()}", file=sys.stderr)
if newValue:
print(
f" + new parameter: '{newValue}'\n"
" + action: Replace the parameter unless correct",
file=sys.stderr,
)
else:
print(
" + action: Remove the parameter unless correct",
file=sys.stderr,
)
if gempa:
gempaStatement(gempa)
# test values
for oldValue, new in valuesCfgFiltered.items():
newValue = new[0]
parameter = new[1]
gempa = new[2]
result = findValue(configFile, oldValue)
lineNo = result[0]
line = result[1]
if not lineNo:
continue
issuesFound += 1
print(
f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated "
f"parameter value '{oldValue}'",
file=sys.stderr,
)
print(f" + full line: {line.rstrip()}", file=sys.stderr)
if newValue:
print(
f" + new value/action: {newValue}",
file=sys.stderr,
)
else:
print(
" + action: Replace the value unless correct",
file=sys.stderr,
)
if gempa:
gempaStatement(gempa)
print("\nTesting bindings configuration:", file=sys.stderr)
# test bindings configurations in key directory
for _, subDirs, _ in os.walk(keysSC):
# skip key files, just consider the module bindings directories
for subDir in subDirs:
bindingDir = os.path.join(keysSC, subDir)
if not os.path.isdir(bindingDir) or not os.listdir(bindingDir):
continue
for config in os.listdir(bindingDir):
bindingFile = os.path.join(bindingDir, config)
if not os.path.isfile(bindingFile):
continue
logging.debug(f"testing bindings in {bindingFile}")
print(f" + file {bindingFile}", file=sys.stderr)
for oldValue, parameter in parametersFiltered.items():
newValue = parameter[0]
gempa = parameter[2]
result = checkParameter(bindingFile, oldValue, newValue)
lineNo = result[0]
line = result[1]
if not lineNo:
continue
issuesFound += 1
print(
f" + POTENIAL ISSUE on line {lineNo}: "
f"obsolete/deprecated parameter '{oldValue}'",
file=sys.stderr,
)
print(f" + full line: {line.rstrip()}", file=sys.stderr)
if newValue:
print(
f" + new parameter: '{newValue}'",
file=sys.stderr,
)
else:
print(
" + - remove it",
file=sys.stderr,
)
if gempa:
print(
f""" + This parameter seems to belong to the \
gempa module '{gempa}'
The proposed action is valid for the most recent version.
Read the module changelog before applying the changes!""",
file=sys.stderr,
)
# test values in bindings
for oldValue, new in valuesCfgFiltered.items():
newValue = new[0]
parameter = new[1]
gempa = new[2]
result = findValue(bindingFile, oldValue, parameter)
lineNo = result[0]
line = result[1]
if not lineNo:
continue
issuesFound += 1
print(
f" + POTENIAL ISSUE on line {lineNo} for parameter "
f"{parameter}:\n"
f" + full line: {line}"
f" + obsolete/deprecated value '{oldValue}' ",
file=sys.stderr,
)
if newValue:
print(
f" + new value: '{newValue}'",
file=sys.stderr,
)
else:
print(
f" + POTENIAL ISSUE on line {lineNo}: obsolete/"
f"deprecated parameter value '{oldValue}' - remove it",
file=sys.stderr,
)
if gempa:
print(
f""" + this parameter seems to belong to the \
gempa module '{gempa}'
The proposed action is valid for the most recent version.
Read the module changelog before applying the changes!""",
file=sys.stderr,
)
print("\nSummary:", file=sys.stderr)
printFinal(self._seiscompVersion, issuesFound)
return True
def main(argv):
app = CheckConfig(len(argv), argv)
return app()
if __name__ == "__main__":
sys.exit(main(sys.argv))