#!/usr/bin/env seiscomp-python ############################################################################ # Copyright (C) 2021 by gempa GmbH # # # # All Rights Reserved. # # # # NOTICE: All information contained herein is, and remains # # the property of gempa GmbH and its suppliers, if any. The intellectual # # and technical concepts contained herein are proprietary to gempa GmbH # # and its suppliers. # # Dissemination of this information or reproduction of this material # # is strictly forbidden unless prior written permission is obtained # # from gempa GmbH. # # # # Author: Dirk Roessler # # Email: roessler@gempa.de # ############################################################################ from __future__ import absolute_import, division, print_function import fnmatch import sys import os import seiscomp.client import seiscomp.io import seiscomp.datamodel import seiscomp.logging as logging # define the latest version for which this script applies latestVersion = "6.6.0" # define module and binding configuration parameters # define module and binding configuration parameter values ############################################################################### # binding parameters parameters = { ########################################################################### # bindigng parameter names # magnitudes # MLh "MLh.maxavg, version:4.5.0": "amplitudes.MLh.params", "MLh.ClippingThreshold, version:4.5.0": "amplitudes.MLh.ClippingThreshold", "MLh.params, version:4.5.0": "magnitudes.MLh.params", # md "md.maxavg, version:4.5.0": "magnitudes.md.seismo", "md.taper, version:4.5.0": "magnitudes.md.taper", "md.signal_length, version:4.5.0": "magnitudes.md.signal_length", "md.butterworth, version:4.5.0": "magnitudes.md.butterworth", "md.depthmax, version:4.5.0": "magnitudes.md.depthmax", "md.deltamax, version:4.5.0": "magnitudes.md.deltamax", "md.snrmin, version:4.5.0": "magnitudes.md.snrmin", "md.mdmax, version:4.5.0": "magnitudes.md.mdmax", "md.fma, version:4.5.0": "magnitudes.md.fma", "md.fmb, version:4.5.0": "magnitudes.md.fmb", "md.fmd, version:4.5.0": "magnitudes.md.fmd", "md.fmf, version:4.5.0": "magnitudes.md.fmf", "md.fmz, version:4.5.0": "magnitudes.md.fmz", "md.linearcorrection, version:4.5.0": "magnitudes.md.linearcorrection", "md.offset, version:4.5.0": "magnitudes.md.offset", "md.stacor, version:4.5.0": "magnitudes.md.stacor", # MLr "MLr.maxavg, version:4.5.0": "magnitudes.MLr.params", # Ms_20 "Ms_20.lowerPeriod, version:4.5.0": "magnitudes.Ms_20.lowerPeriod", "Ms_20.upperPeriod, version:4.5.0": "magnitudes.Ms_20.upperPeriod", "Ms_20.minDist, version:4.5.0": "magnitudes.Ms_20.minDist", "Ms_20.maxDist, version:4.5.0": "magnitudes.Ms_20.maxDist", "Ms_20.maxDepth, version:4.5.0": "magnitudes.Ms_20.maxDepth", # MLv "MLv.logA0, version:4.5.0": "magnitudes.MLv.logA0", "MLv.maxDistanceKm, version:4.5.0": "magnitudes.MLv.maxDistanceKm", # ML "ML.logA0, version:4.5.0": "magnitudes.ML.logA0", "ML.maxDistanceKm, version:4.5.0": "magnitudes.ML.maxDistanceKm", ########################################################################### ########################################################################### # module parameter names # scmv "legend, version:4.6.0": "scheme.map.showLegends", # data base "database.type, version:5.0.0": "database", "database.parameters, version:5.0.0": "database", # RecordsStream "recordstream.service, version:5.0.0": "recordstream", "recordstream.source, version:5.0.0": "recordstream", # scautoloc "autoloc.locator.profile, version:4.3.0": "locator.profile", "autoloc.cleanupInterval, version:4.3.0": "buffer.cleanupInterval", "autoloc.maxAge, version:4.3.0": "buffer.pickKeep", "autoloc.wakeupInterval, version:4.3.0": "", # magnitudes "magnitudes.*.regions, version:5.0.0": "magnitudes.*.regionFile", ########################################################################### # SC 5.0.0 # global parameters for eventlist "eventlist.customColumn, version:5.0.0": "eventlist.customColumn.name", "eventlist.regions, version:5.0.0": "eventlist.filter.regions.profiles", "eventlist.region, version:5.0.0": "eventlist.filter.regions.region", # global parameters for eventedit "eventedit.customColumn, version:5.0.0": "eventedit.origin.customColumn.name", "eventedit.customColumn.default, version:5.0.0": "eventedit.origin.customColumn.default", "eventedit.customColumn.originCommentID, version:5.0.0": "eventedit.origin.customColumn.originCommentID", "eventedit.customColumn.pos, version:5.0.0": "eventedit.origin.customColumn.pos", "eventedit.customColumn.colors, version:5.0.0": "eventedit.origin.customColumn.colors", ########################################################################### # SC 5.1.1 # scolv "picker.auxilliary., version:5.1.1": "picker.auxiliary.", # SC 5.4.0 # ttt.homogeneous "ttt.homogeneous.profile., version:5.4.0": "ttt.homogeneous.", # SC 6.0.0 # scardac "batchSize, version:6.0.0": "", # SC 6.5.0 # StdLoc "GridSearch.cellSize, version:6.5.0": "GridSearch.numPoints", # scanloc "clusterSearch.ignorePicks, version:scanloc": "buffer.ignorePickTimeDifference", "clusterSearch.ignorePickTimeDifference, version:scanloc": "buffer.ignorePickTimeDifference", "buffer.originAuthorWhiteList, version:scanloc": "buffer.authorWhiteList", "score.weight.p, version:scanloc": "score.sum.weight.p", "score.weight.s, version:scanloc": "score.sum.weight.s", "score.weight.p0, version:scanloc": "score.sum.weight.p0", "score.weight.s0, version:scanloc": "score.sum.weight.s0", "score.weight.residual, version:scanloc": "score.sum.weight.residual", "score.weight.depth, version:scanloc": "score.sum.weight.depth", "score.weight.increaseManual, version:scanloc": "score.sum.weight.increaseManual", "ossum.p, version:scanloc": "score.sum.weight.p", "ossum.s, version:scanloc": "score.sum.weight.s", "ossum.p0, version:scanloc": "score.sum.weight.p0", "ossum.s0, version:scanloc": "score.sum.weight.s0", "ossum.residual, version:scanloc": "score.sum.weight.residual", "ossum.depth, version:scanloc": "score.sum.weight.depth", "ossum.increaseManual, version:scanloc": "score.sum.weight.increaseManual", "ossum.normalizationDepth, version:scanloc": "score.sum.weight.normalizationDepth", "ossum.normalizationRMS, version:scanloc": "score.sum.weight.normalizationRMS", "clustersearch.streamCheckLevel, version:scanloc": "streamCheckLevel", # vortex "script.*.path, version:vortex": "event.script.*.path", # TOAST "alwaysShowAdditionalOptions, version:toast": "dissemination.alwaysShowAdditionalOptions", "confirmationRequired, version:toast": "dissemination.confirmationRequired", "disseminationSelectionBehavior, version:toast": "dissemination.selectionBehavior", } ############################################################################### # parameter values # module and binding configuration valuesCfg = { # plugins "parameter:plugins,rsas, version:4.5.0": "", # messaging "parameter:plugins,dbplugin, version:4.0.0": "", # events list "parameter:eventlist.visibleColumns, TP, version:4.0.4": "MType for \ eventlist.visibleColumns", # magnitude calibration functions "parameter:logA0,0-1., version:5.0.0": "New format for logA0 parameter of \ magnitude: delta1:M1,delta2:M2,...", # sigma "parameter:plugins,gmpeeewd, version:sigma": "Replace 'gmpeeewd' by 'cppgmpe'", "parameter:plugins,py3gmpe, version:sigma": "Replace 'py3gmpe' by 'cppgmpe' unless \ more Python3 GMPEs are needed", "parameter:gempa.plugins,py3gmpe, version:sigma": "Remove and apply defaults or \ replace 'py3gmpe' by 'cppgmpe' unless more Python3 GMPEs are needed. \ Check cppgmpe/GMPE names!", "parameter:gempa.plugins,pygmpe, version:sigma": "Remove and apply defaults or \ replace 'pygmpe' by 'cppgmpe' unless more Python GMPEs are needed. \ Check cppgmpe/GMPE names!", "parameter:plugins,pygmpe, version:sigma": "Replace 'pygmpe' by 'cppgmpe' unless \ more Python GMPEs are needed", "gfz-potsdam.de, version:4.0.0": "Replace by gfz.de - server address has changed", } ############################################################################### def gempaStatement(gempa): print( f""" + This parameter seems to belong to the gempa module '{gempa}'. The \ proposed action is valid for the most recent version. Read the changelog of '{gempa}' before applying \ the changes!""", file=sys.stderr, ) return True def checkParameter(inFile, oldValue, newValue): found = 0 line = "" with open(inFile, encoding="utf-8") as f: lines = f.readlines() lineNo = 0 for line in lines: found = None lineNo += 1 if line.startswith("#"): continue words = line.split() # Check each possible substring for i in range(len(words)): for j in range(i + 1, len(words) + 1): substring = " ".join(words[i:j]) if fnmatch.fnmatch(substring, oldValue) or oldValue in substring: # return True # if oldValue in line: # try reducing false positives idxStart = line.find(oldValue) if idxStart != 0: if "." == line[idxStart] or "," == line[idxStart]: continue idxEnd = idxStart + len(oldValue) if idxEnd != len(oldValue) - 1: if "." == line[idxEnd] or "," == line[idxEnd]: continue if newValue not in oldValue and newValue not in line: found = lineNo if newValue in oldValue: found = lineNo if found: # print( # f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated \ # parameter {oldValue} - new parameter: {newValue}", # file=sys.stderr, # ) # print(f" + full line: {line.rstrip()}", file=sys.stderr) return found, line return found, line ############################################################################### def findValue(inFile, oldValue, parameter=None): found = 0 line = "" with open(inFile, encoding="utf-8") as f: lines = f.readlines() lineNo = 0 for line in lines: lineNo += 1 if line.startswith("#"): continue if parameter and parameter not in line: continue if oldValue in line: found = lineNo return found, line return found, line ############################################################################### def printFinal(version, issuesFound=None): print( f"This check applies to SeisComP in version <= {version}", file=sys.stderr, ) print(" + read your own version, e.g.: 'seiscomp exec scm -V'", file=sys.stderr) if issuesFound: print(f" + found issues: {issuesFound}", file=sys.stderr) else: print(" + no issues found \U0001f44d", file=sys.stderr) print( """ Alert and recommendation: * Applies to: SeisComP databases using the charset utf8mb4 created with SeisComP in version 6.0.0 <= version < 6.7.0 or nightly after 17 August 2023 until February 2025 * Issue: The charset used for the database does not discriminate upper and lower case characters. * Actions: * Install the script 'gempa-check-database' with the package 'seiscomp-tools' using gsm or download from https://data.gempa.de/packages/Public/. * Stop scmaster * Ensure, no other modules like scdb, scardac, etc. or custom tools from internal or external clients attempt accessing the database. * Login to your database, e.g.: mysql -u sysop -p * Source one of the suggested update scripts: SOURCE /tmp/update-mysql-charset-CHARACTERSET-DATABASE.sql""" ) return True class CheckConfig(seiscomp.client.Application): def __init__(self, argc, argv): seiscomp.client.Application.__init__(self, argc, argv) self.setMessagingEnabled(False) self.setDatabaseEnabled(False, False) self._configDir = False self._home = os.environ["HOME"] self._root = None self._seiscompVersion = latestVersion def createCommandLineDescription(self): self.commandline().addGroup("SeisComP") self.commandline().addStringOption( "SeisComP", "config-dir,c", "Path to non-standard @CONFIGDIR@. Default: False", ) self.commandline().addStringOption( "SeisComP", "root,r", "SeisComP root directory to search for " "SYSTEMCONFIGDIR. Default: $SEISCOMP_ROOT", ) self.commandline().addStringOption( "SeisComP", "seiscomp-version, s", "SeisComP version number to be considered for testing.", ) def printUsage(self): print( f"""Usage: {os.path.basename(__file__)} [options] Identify + report legacy configuration for SeisComP and gempa modules in version <= \ {latestVersion}""" ) seiscomp.client.Application.printUsage(self) print( f"""Examples: For the test results read the log file or use --console 1 Simple run considering all parameters up version {self._seiscompVersion} {os.path.basename(__file__)} Run with specfic $SEISCOMP_ROOT directory and SeisComP version {os.path.basename(__file__)} -r /home/sysop/seisomp-test --console 1 \ --seiscomp-version 4.8.0""" ) def init(self): if not seiscomp.client.Application.init(self): return False try: self._root = self.commandline().optionString("root") except RuntimeError: try: self._root = os.environ["SEISCOMP_ROOT"] except KeyError: print( "SEISCOMP_ROOT directory is undefined. Cannot continue.", file=sys.stderr, ) return False logging.debug(f"ROOT directory not given. Assuming {self._root}") try: self._configDir = self.commandline().optionString("config-dir") except RuntimeError: logging.debug( f"Configuration directory not set. Creating from ROOT: {self._root}" ) try: self._seiscompVersion = self.commandline().optionString("seiscomp-version") except RuntimeError: logging.debug(f"SeisComP not set. Assuming version {self._seiscompVersion}") return True def run(self): issuesFound = 0 if not os.path.exists(self._home): print( f"{self._home} does not exist, check your option --home", file=sys.stderr, ) return False # define @CONFIGDIR@ / @SYSTEMCONFIGDIR@ if not self._configDir: configSC = os.path.join(self._home, ".seiscomp") configSC3 = os.path.join(self._home, ".seiscomp3") systemConfigSC = os.path.join(self._root, "etc") keysSC = os.path.join(self._root, "etc/key") else: configSC = self._configDir configSC3 = configSC systemConfigSC = os.path.join(configSC, "etc") keysSC = os.path.join(configSC, "etc/key") print("\nTesting general issues:", file=sys.stderr) if os.path.exists(configSC3): for configFile in os.listdir(configSC3): if configFile.endswith("cfg"): print( "SeisComP3 configuration still exists in " f"{configSC3} - consider migrating or removing", file=sys.stderr, ) issuesFound += 1 break if not os.path.exists(configSC): logging.debug( f"SeisComP configuration '{configSC}' does not exist in '{self._home}'" "and will not be tested. You may consider using '-c'." ) return False if not os.path.exists(systemConfigSC): logging.error( f"SeisComP system configuration {systemConfigSC} does not exist in " f"{self._root}. Set your SeisComP variables or consider using '-r'." ) return False if not os.path.exists(keysSC): logging.error( f"SeisComP key directory {keysSC} does not exist in {self._root}. " "Set your SeisComP variables or consider using '-r'" ) return False # test if old license path exists oldLicenses = os.path.join(configSC, "licenses") if os.path.exists(oldLicenses): print( f" + ISSUE: Found old license directory '{oldLicenses}'. Move it to " "@DATADIR@/licenses", file=sys.stderr, ) print( " + more information: " "'https://www.gempa.de/news/2022/02/licenses/'", file=sys.stderr, ) issuesFound += 1 # filter parameters by considered version parametersFiltered = {} for test, newValue in parameters.items(): items = test.split(",") oldValue = None version = None parameter = None gempa = False for item in items: if "version" in item: version = item.split(":")[1] if len(version.split(".")) != 3: gempa = version continue oldValue = item if oldValue is None: continue if not gempa and version and version > self._seiscompVersion: continue parametersFiltered[oldValue] = [newValue, parameter, gempa] # if ", version:" in oldValue: # key = oldValue.split(", version:")[0] # version = oldValue.split(", version:")[1] # if version <= self._seiscompVersion: # parametersFiltered[key] = newValue # else: # parametersFiltered[oldValue] = newValue # module configuration parameter values valuesCfgFiltered = {} for oldValue, newValue in valuesCfg.items(): items = oldValue.split(",") key = None version = None parameter = None gempa = False for item in items: if "version" in item: version = item.split(":")[1] if len(version.split(".")) != 3: gempa = version elif "parameter" in item: parameter = item.split(":")[1] else: key = item if key is None: continue if not gempa and version and version > self._seiscompVersion: continue valuesCfgFiltered[key] = [newValue, parameter, gempa] print("\nTesting module configurations in @CONFIGDIR@:", file=sys.stderr) # test module configurations in CONFIGDIR for config in os.listdir(configSC): configFile = os.path.join(configSC, config) if not os.path.isfile(configFile): continue if not configFile.endswith("cfg"): continue logging.debug(f" + testing module configurations in {configFile}") print(f" + file {configFile}", file=sys.stderr) # test parameters for oldValue, parameter in parametersFiltered.items(): newValue = parameter[0] gempa = parameter[2] result = checkParameter(configFile, oldValue, newValue) lineNo = result[0] line = result[1] if not lineNo: continue issuesFound += 1 print( f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated " f"parameter '{oldValue}'", file=sys.stderr, ) print(f" + full line: {line.rstrip()}", file=sys.stderr) if newValue: print( f" + new parameter: '{newValue}'", file=sys.stderr, ) else: print( " + action: Remove the value", file=sys.stderr, ) if gempa: gempaStatement(gempa) # test values for oldValue, new in valuesCfgFiltered.items(): newValue = new[0] parameter = new[1] gempa = new[2] result = findValue(configFile, oldValue) lineNo = result[0] line = result[1] if not lineNo: continue issuesFound += 1 if newValue: print( f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated " f"parameter '{oldValue}' - new parameter: '{newValue}'\n" " + action: Rename the parameter unless correct", file=sys.stderr, ) print(f" + full line: {line.rstrip()}", file=sys.stderr) continue print( f" + POTENIAL ISSUE on line lineNo: obsolete/deprecated " f"parameter '{oldValue}'\n" " + action: Remove the parameter unless correct", file=sys.stderr, ) print(f" + full line: {line.rstrip()}", file=sys.stderr) if gempa: gempaStatement(gempa) if issuesFound == 0: print(" + found no issue", file=sys.stderr) # test module configurations in SYSTEMCONFIGDIR print(f"\nTesting module configurations in {systemConfigSC}:", file=sys.stderr) for config in os.listdir(systemConfigSC): configFile = os.path.join(systemConfigSC, config) if not os.path.isfile(configFile): continue if not configFile.endswith("cfg"): continue logging.debug(f"testing module configurations in {configFile}") print(f" + file {configFile}", file=sys.stderr) # test parameters for oldValue, parameter in parametersFiltered.items(): newValue = parameter[0] gempa = parameter[2] result = checkParameter(configFile, oldValue, newValue) lineNo = result[0] line = result[1] if not lineNo: continue issuesFound += 1 print( f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated " f"parameter '{oldValue}'", file=sys.stderr, ) print(f" + full line: {line.rstrip()}", file=sys.stderr) if newValue: print( f" + new parameter: '{newValue}'\n" " + action: Replace the parameter unless correct", file=sys.stderr, ) else: print( " + action: Remove the parameter unless correct", file=sys.stderr, ) if gempa: gempaStatement(gempa) # test values for oldValue, new in valuesCfgFiltered.items(): newValue = new[0] parameter = new[1] gempa = new[2] result = findValue(configFile, oldValue) lineNo = result[0] line = result[1] if not lineNo: continue issuesFound += 1 print( f" + POTENIAL ISSUE on line {lineNo}: obsolete/deprecated " f"parameter value '{oldValue}'", file=sys.stderr, ) print(f" + full line: {line.rstrip()}", file=sys.stderr) if newValue: print( f" + new value/action: {newValue}", file=sys.stderr, ) else: print( " + action: Replace the value unless correct", file=sys.stderr, ) if gempa: gempaStatement(gempa) print("\nTesting bindings configuration:", file=sys.stderr) # test bindings configurations in key directory for _, subDirs, _ in os.walk(keysSC): # skip key files, just consider the module bindings directories for subDir in subDirs: bindingDir = os.path.join(keysSC, subDir) if not os.path.isdir(bindingDir) or not os.listdir(bindingDir): continue for config in os.listdir(bindingDir): bindingFile = os.path.join(bindingDir, config) if not os.path.isfile(bindingFile): continue logging.debug(f"testing bindings in {bindingFile}") print(f" + file {bindingFile}", file=sys.stderr) for oldValue, parameter in parametersFiltered.items(): newValue = parameter[0] gempa = parameter[2] result = checkParameter(bindingFile, oldValue, newValue) lineNo = result[0] line = result[1] if not lineNo: continue issuesFound += 1 print( f" + POTENIAL ISSUE on line {lineNo}: " f"obsolete/deprecated parameter '{oldValue}'", file=sys.stderr, ) print(f" + full line: {line.rstrip()}", file=sys.stderr) if newValue: print( f" + new parameter: '{newValue}'", file=sys.stderr, ) else: print( " + - remove it", file=sys.stderr, ) if gempa: print( f""" + This parameter seems to belong to the \ gempa module '{gempa}' The proposed action is valid for the most recent version. Read the module changelog before applying the changes!""", file=sys.stderr, ) # test values in bindings for oldValue, new in valuesCfgFiltered.items(): newValue = new[0] parameter = new[1] gempa = new[2] result = findValue(bindingFile, oldValue, parameter) lineNo = result[0] line = result[1] if not lineNo: continue issuesFound += 1 print( f" + POTENIAL ISSUE on line {lineNo} for parameter " f"{parameter}:\n" f" + full line: {line}" f" + obsolete/deprecated value '{oldValue}' ", file=sys.stderr, ) if newValue: print( f" + new value: '{newValue}'", file=sys.stderr, ) else: print( f" + POTENIAL ISSUE on line {lineNo}: obsolete/" f"deprecated parameter value '{oldValue}' - remove it", file=sys.stderr, ) if gempa: print( f""" + this parameter seems to belong to the \ gempa module '{gempa}' The proposed action is valid for the most recent version. Read the module changelog before applying the changes!""", file=sys.stderr, ) print("\nSummary:", file=sys.stderr) printFinal(self._seiscompVersion, issuesFound) return True def main(argv): app = CheckConfig(len(argv), argv) return app() if __name__ == "__main__": sys.exit(main(sys.argv))