Files
2025/bin/dump_picks

765 lines
27 KiB
Plaintext
Executable File

#!/usr/bin/env seiscomp-python
############################################################################
# Copyright (C) 2016 by gempa GmbH #
# #
# All Rights Reserved. #
# #
# NOTICE: All information contained herein is, and remains #
# the property of gempa GmbH and its suppliers, if any. The intellectual #
# and technical concepts contained herein are proprietary to gempa GmbH #
# and its suppliers. #
# Dissemination of this information or reproduction of this material #
# is strictly forbidden unless prior written permission is obtained #
# from gempa GmbH. #
# #
# Author: Enrico Ellguth, Dirk Roessler #
# Email: enrico.ellguth@gempa.de, roessler@gempa.de #
############################################################################
import datetime
import os
import sys
from seiscomp import core, datamodel, io
from seiscomp.client import Application
from seiscomp import geo
def str2time(timestring):
"""
Liberally accept many time string formats and convert them to a
seiscomp.core.Time
"""
timestring = timestring.strip()
for c in ["-", "/", ":", "T", "Z"]:
timestring = timestring.replace(c, " ")
timestring = timestring.split()
assert 3 <= len(timestring) <= 6
timestring.extend((6 - len(timestring)) * ["0"])
timestring = " ".join(timestring)
fmt = "%Y %m %d %H %M %S"
if timestring.find(".") != -1:
fmt += ".%f"
t = core.Time()
t.fromString(timestring, fmt)
return t
def utc():
return datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
class DumpPicks(Application):
def __init__(self, argc, argv):
Application.__init__(self, argc, argv)
self.output = "-"
self.type = "0"
self.margin = [300]
self.originID = None
self.bbox = None
self.noamp = False
self.automatic = False
self.manual = False
self.checkInventory = False
self.author = None
self.hours = None
self.minutes = None
self.start = None
self.end = None
self.network = None
self.station = None
self.tmin = str2time("1970-01-01 00:00:00")
self.tmax = str2time(str(utc()))
self.delay = None
self.setMessagingEnabled(False)
self.setDatabaseEnabled(True, True)
def createCommandLineDescription(self):
self.commandline().addGroup("Dump")
self.commandline().addStringOption(
"Dump",
"hours",
"Start search hours before now considering object time, not creation time. "
"If --minutes is given as well they will be added. "
"If set, --time-window, --start, --end are ignored.",
)
self.commandline().addStringOption(
"Dump",
"minutes",
"Start search minutes before now considering object time, not creation time. "
"If --hours is given as well they will be added. "
"If set, --time-window, --start, --end are ignored.",
)
self.commandline().addStringOption(
"Dump",
"start",
"Start time of search until now considering object time, not creation time."
" If set, --time-window is ignored.",
)
self.commandline().addStringOption(
"Dump",
"end",
"End time of search considering object time, not creation time. If set, "
"--time-window is ignored.",
)
self.commandline().addStringOption(
"Dump",
"time-window,t",
"Specify time window to search picks and amplitudes by their time. Use one "
"single string which must be enclosed by quotes in case of spaces in the "
"time string. Times are of course in UTC and separated by a tilde '~'. "
"Uses: 1970-01-01 00:00:00 to now if not set.",
)
self.commandline().addStringOption(
"Dump",
"maximum-delay",
"Maximum allowed delay of picks or amplitudes, hence the difference between"
" creation time and actual time value. Allows identifcation of picks found "
"in real time.",
)
self.commandline().addStringOption(
"Dump",
"region,r",
"Dump picks only from sensors in given region. Implies loading an "
"inventory.\n"
"Format: minLat,minLon,maxLat,maxLon \n"
"Default: -90,-180,90,180 if not set.",
)
self.commandline().addOption(
"Dump",
"check-inventory,c",
"Dump picks only when corresponding streams are found in inventory.",
)
self.commandline().addStringOption(
"Dump",
"origin,O",
"Origin ID. Dump all "
"picks associated with the origin that has the given origin ID.",
)
self.commandline().addOption("Dump", "manual,m", "Dump only manual picks.")
self.commandline().addOption(
"Dump", "automatic,a", "Dump only automatic picks."
)
self.commandline().addOption(
"Dump",
"no-amp,n",
"Do not dump amplitudes from picks. "
"Amplitudes are not required by scanloc.",
)
self.commandline().addStringOption(
"Dump", "author", "Filter picks by the given author."
)
self.commandline().addStringOption(
"Dump",
"net-sta",
"Filter picks and amplitudes by given network code or "
"network and station code. Format: NET or NET.STA.",
)
self.commandline().addGroup("Output")
self.commandline().addStringOption(
"Output",
"output,o",
"Name of output file. If not given, all data is written to stdout.",
)
self.commandline().addStringOption(
"Output",
"type",
f"Type of output format. Default: {self.type}.\n"
"0 / scml: SCML containing all objects (default if option is not used)\n"
"1 / streams: Time windows and streams for all picks like in scevtstreams\n"
"2 / caps: Time windows and streams in capstool format\n"
"3 / fdsnws: Time windows and streams in FDSN dataselect webservice POST \
format\n"
"Except for type 0, only picks are considered ignoring all other objects.",
)
self.commandline().addOption(
"Output",
"formatted,f",
"Output formatted XML. Default is unformatted. Applies only for type 0.",
)
self.commandline().addStringOption(
"Output",
"margin",
"Time margin applied around pick times along with --type = [1:]. Use 2 "
"comma-separted values (before,after) for asymmetric margins, e.g. "
f"--margin 120,300. Default: {self.margin[0]} s.",
)
def printUsage(self):
print(
f"""Usage:
{os.path.basename(__file__)} [options]
Read picks and amplitudes from database and dump them to a file or to standard output.\
"""
)
Application.printUsage(self)
print(
f"""Examples:
Dump all picks within a region and a period of time
{os.path.basename(__file__)} -d localhost -t 2023-01-20T13:52:00~2023-01-20T13:57:00\
-r "-10,-90,10,120"
Search 24 hours before now for automatic picks from author "scautopick" with low delay \
ignoring amplitudes
{os.path.basename(__file__)} -d localhost --hours 24 -a -n --author "scautopick" \
--maximum-delay 60
Dump the streams of picks with time windows fetching the corresponding data from a \
local CAPS server
{os.path.basename(__file__)} -d localhost --type 2 --margin 60 | capstool \
-H localhost -o data.mseed
Dump the streams of picks with time windows fetching the corresponding data from a \
local SDS archive
{os.path.basename(__file__)} -d localhost --type 1 --margin 60 | scart -dsE -l - \
/archive -o data.mseed
"""
)
def init(self):
if not Application.init(self):
return False
try:
self.output = self.commandline().optionString("output")
except RuntimeError:
pass
try:
self.type = self.commandline().optionString("type")
except RuntimeError:
pass
if self.type == "scml":
self.type = "0"
elif self.type == "streams":
self.type = "1"
elif self.type == "caps":
self.type = "2"
elif self.type == "fdsnws":
self.type = "3"
try:
self.margin = self.commandline().optionString("margin").split(",")
except RuntimeError:
pass
try:
self.originID = self.commandline().optionString("origin")
except RuntimeError:
pass
if not self.originID:
try:
boundingBox = self.commandline().optionString("region")
self.bbox = boundingBox.split(",")
if len(self.bbox) != 4:
print(
"Invalid region given, expected lat0,lon0,lat1,lon1",
file=sys.stderr,
)
return False
self.bbox[0] = str(geo.GeoCoordinate.normalizeLat(float(self.bbox[0])))
self.bbox[1] = str(geo.GeoCoordinate.normalizeLon(float(self.bbox[1])))
self.bbox[2] = str(geo.GeoCoordinate.normalizeLat(float(self.bbox[2])))
self.bbox[3] = str(geo.GeoCoordinate.normalizeLon(float(self.bbox[3])))
self.checkInventory = True
except RuntimeError:
boundingBox = "-90,-180,90,180"
self.bbox = boundingBox.split(",")
print("Settings", file=sys.stderr)
print(
f" + considered region: {self.bbox[0]} - {self.bbox[2]} deg North, "
f"{self.bbox[1]} - {self.bbox[3]} deg East",
file=sys.stderr,
)
try:
self.hours = float(self.commandline().optionString("hours"))
except RuntimeError:
pass
try:
self.minutes = float(self.commandline().optionString("minutes"))
except RuntimeError:
pass
try:
self.start = self.commandline().optionString("start")
except RuntimeError:
pass
try:
self.end = self.commandline().optionString("end")
except RuntimeError:
pass
delta = 0.0
if self.hours:
delta = self.hours * 60
if self.minutes:
delta += self.minutes
if self.hours or self.minutes:
print(
" + time window set by hours and/or minutes option: ignoring all "
"other time parameters",
file=sys.stderr,
)
dt = datetime.timedelta(minutes=delta)
self.tmin = str2time(str(utc() - dt))
self.tmax = str2time(str(utc()))
self.start = None
self.end = None
else:
if self.start:
print(
" + time window set by start option: ignoring --time-window",
file=sys.stderr,
)
self.tmin = str2time(self.start)
if self.end:
print(
" + time window set by end option: ignoring --time-window",
file=sys.stderr,
)
self.tmax = str2time(self.end)
if not self.start and not self.end:
try:
self.tmin, self.tmax = map(
str2time,
self.commandline().optionString("time-window").split("~"),
)
print(
" + time window set by time-window option", file=sys.stderr
)
except RuntimeError:
print(
" + no time window given exlicitly: Assuming defaults",
file=sys.stderr,
)
print(
f" + considered time window: {str(self.tmin)} - {str(self.tmax)}",
file=sys.stderr,
)
else:
print(
" + searching for picks is based on originID, ignoring "
"region and time window",
file=sys.stderr,
)
try:
self.delay = float(self.commandline().optionString("maximum-delay"))
except RuntimeError:
pass
if not self.checkInventory:
self.checkInventory = self.commandline().hasOption("check-inventory")
if self.checkInventory:
print(
" + dumping only picks for streams found in inventory", file=sys.stderr
)
else:
print(
" + do not consider inventory information for dumping picks",
file=sys.stderr,
)
if self.commandline().hasOption("no-amp"):
self.noamp = True
else:
self.noamp = False
if self.type != "0":
self.noamp = True
if self.noamp:
print(" + dumping picks without amplitudes", file=sys.stderr)
else:
print(" + dumping picks with amplitudes", file=sys.stderr)
if self.commandline().hasOption("manual"):
self.manual = True
print(" + dumping only manual objects", file=sys.stderr)
else:
self.manual = False
print(" + considering also manual objects", file=sys.stderr)
if self.commandline().hasOption("automatic"):
if not self.manual:
self.automatic = True
print(" + dumping only automatic picks", file=sys.stderr)
else:
print(
"EXIT - Script was started with competing options -a and -m",
file=sys.stderr,
)
return False
else:
self.automatic = False
print(" + considering also automatic objects", file=sys.stderr)
try:
self.author = self.commandline().optionString("author")
except RuntimeError:
pass
networkStation = None
try:
networkStation = self.commandline().optionString("net-sta")
print(
f" + filter objects by network / station code: {networkStation}",
file=sys.stderr,
)
except RuntimeError:
pass
if networkStation:
try:
self.network = networkStation.split(".")[0]
except IndexError:
print(
f"Error in network code '{networkStation}': Use '--net-sta' with "
"format NET or NET.STA",
file=sys.stderr,
)
return False
try:
self.station = networkStation.split(".")[1]
except IndexError:
print(
f" + no station code given in '--net-sta {networkStation}' - "
"using all stations from network",
file=sys.stderr,
)
return True
def run(self):
db = self.database()
def _T(name):
return db.convertColumnName(name)
def _time(time):
return db.timeToString(time)
colLat, colLon = _T("latitude"), _T("longitude")
dbq = self.query()
ep = datamodel.EventParameters()
picks = []
noAmps = 0
if self.originID:
for p in dbq.getPicks(self.originID):
picks.append(datamodel.Pick.Cast(p))
for p in picks:
dbq.loadComments(p)
ep.add(p)
if not self.noamp:
for a in dbq.getAmplitudesForOrigin(self.originID):
amp = datamodel.Amplitude.Cast(a)
ep.add(amp)
else:
fmt = "%Y-%m-%d %H:%M:%S"
if self.checkInventory:
q = (
"select distinct(PPick.%s), Pick.* "
"from PublicObject as PPick, Pick, Network, Station, SensorLocation "
"where PPick._oid=Pick._oid and Network._oid=Station._parent_oid and "
"Station._oid=SensorLocation._parent_oid and Station.%s >= %s and "
"Station.%s <= %s and Station.%s >= %s and Station.%s <= %s and "
"SensorLocation.%s=Pick.%s and SensorLocation.%s <= Pick.%s and "
"(SensorLocation.%s is null or SensorLocation.%s > Pick.%s) and "
"Station.%s=Pick.%s and Network.%s=Pick.%s and "
"Pick.%s >= '%s' and Pick.%s < '%s'"
""
% (
_T("publicID"),
colLat,
self.bbox[0],
colLat,
self.bbox[2],
colLon,
self.bbox[1],
colLon,
self.bbox[3],
_T("code"),
_T("waveformID_locationCode"),
_T("start"),
_T("time_value"),
_T("end"),
_T("end"),
_T("time_value"),
_T("code"),
_T("waveformID_stationCode"),
_T("code"),
_T("waveformID_networkCode"),
_T("time_value"),
self.tmin.toString(fmt),
_T("time_value"),
self.tmax.toString(fmt),
)
)
else:
q = (
"select distinct(PPick.%s), Pick.* "
"from PublicObject as PPick, Pick "
"where PPick._oid=Pick._oid and "
"Pick.%s >= '%s' and Pick.%s < '%s'"
""
% (
_T("publicID"),
_T("time_value"),
self.tmin.toString(fmt),
_T("time_value"),
self.tmax.toString(fmt),
)
)
if self.manual:
q = q + f" and Pick.{_T('evaluationMode')} = 'manual' "
if self.automatic:
q = q + f" and Pick.{_T('evaluationMode')} = 'automatic' "
if self.author:
q = q + f" and Pick.{_T('creationInfo_author')} = '{self.author}' "
if self.network:
q = q + f" and Pick.{_T('waveformID_networkCode')} = '{self.network}' "
if self.station:
q = q + f" and Pick.{_T('waveformID_stationCode')} = '{self.station}' "
for p in dbq.getObjectIterator(q, datamodel.Pick.TypeInfo()):
pick = datamodel.Pick.Cast(p)
if (
self.delay
and float(pick.creationInfo().creationTime() - pick.time().value())
> self.delay
):
continue
picks.append(pick)
for p in picks:
dbq.loadComments(p)
ep.add(p)
if not self.noamp:
if self.checkInventory:
q = (
"select distinct(PAmplitude.%s), Amplitude.* "
"from PublicObject as PAmplitude, Amplitude, PublicObject \
as PPick, Pick, Network, Station, SensorLocation "
"where PAmplitude._oid=Amplitude._oid and "
"PPick._oid=Pick._oid and Network._oid=Station._parent_oid and "
"Station._oid=SensorLocation._parent_oid and Station.%s >= %s and "
"Station.%s <= %s and Station.%s >= %s and Station.%s <= %s and "
"SensorLocation.%s=Pick.%s and SensorLocation.%s <= Pick.%s and "
"(SensorLocation.%s is null or SensorLocation.%s > Pick.%s) and "
"Station.%s=Pick.%s and Network.%s=Pick.%s and "
"Pick.%s >= '%s' and Pick.%s < '%s' and PPick.%s=Amplitude.%s"
""
% (
_T("publicID"),
colLat,
self.bbox[0],
colLat,
self.bbox[2],
colLon,
self.bbox[1],
colLon,
self.bbox[3],
_T("code"),
_T("waveformID_locationCode"),
_T("start"),
_T("time_value"),
_T("end"),
_T("end"),
_T("time_value"),
_T("code"),
_T("waveformID_stationCode"),
_T("code"),
_T("waveformID_networkCode"),
_T("time_value"),
self.tmin.toString(fmt),
_T("time_value"),
self.tmax.toString(fmt),
_T("publicID"),
_T("pickID"),
)
)
else:
q = (
"select distinct(PAmplitude.%s), Amplitude.* "
"from PublicObject as PAmplitude, Amplitude, PublicObject as PPick, Pick "
"where PAmplitude._oid=Amplitude._oid and PPick._oid=Pick._oid and "
"Pick.%s >= '%s' and Pick.%s < '%s' and PPick.%s=Amplitude.%s"
""
% (
_T("publicID"),
_T("time_value"),
self.tmin.toString(fmt),
_T("time_value"),
self.tmax.toString(fmt),
_T("publicID"),
_T("pickID"),
)
)
if self.manual:
q = q + f" and Pick.{_T('evaluationMode')} = 'manual' "
if self.automatic:
q = q + f" and Pick.{_T('evaluationMode')} = 'automatic' "
if self.author:
q = q + f" and Pick.{_T('creationInfo_author')} = '{self.author}' "
if self.network:
q = q + " and Pick.%s = '%s' " % (
_T("waveformID_networkCode"),
self.network,
)
if self.station:
q = q + " and Pick.%s = '%s' " % (
_T("waveformID_stationCode"),
self.station,
)
for a in dbq.getObjectIterator(q, datamodel.Amplitude.TypeInfo()):
amp = datamodel.Amplitude.Cast(a)
if (
self.delay
and float(
amp.creationInfo().creationTime()
- amp.timeWindow().reference()
)
> self.delay
):
continue
ep.add(amp)
noAmps += 1
if self.type == "0":
ar = io.XMLArchive()
ar.create(self.output)
ar.setFormattedOutput(self.commandline().hasOption("formatted"))
ar.writeObject(ep)
ar.close()
elif self.type in ["1", "2", "3"]:
if len(picks) == 0:
print(
"No picks are found and written",
file=sys.stderr,
)
return False
# convert times to string depending on requested output format
# time and line format
if self.type == "2":
timeFMT = "%Y,%m,%d,%H,%M,%S"
lineFMT = "{0} {1} {2} {3} {4} {5}"
elif self.type == "3":
timeFMT = "%FT%T"
lineFMT = "{2} {3} {4} {5} {0} {1}"
else:
timeFMT = "%F %T"
lineFMT = "{0};{1};{2}.{3}.{4}.{5}"
lines = set()
for pick in picks:
net = pick.waveformID().networkCode()
station = pick.waveformID().stationCode()
loc = pick.waveformID().locationCode()
channelGroup = f"{pick.waveformID().channelCode()[:2]}*"
# FDSNWS requires empty location to be encoded by 2 dashes
if not loc and self.type == "3":
loc = "--"
# add some marging to picks times
minTime = pick.time().value() - core.TimeSpan(float(self.margin[0]))
maxTime = pick.time().value() + core.TimeSpan(float(self.margin[-1]))
minTime = minTime.toString(timeFMT)
maxTime = maxTime.toString(timeFMT)
lines.add(
lineFMT.format(minTime, maxTime, net, station, loc, channelGroup)
)
if self.output == "-":
out = sys.stdout
else:
print(f"Output data to file: {self.output}", file=sys.stderr)
try:
out = open(self.output, "w", encoding="utf8")
except Exception:
print("Cannot create output file '{self.output}'", file=sys.stderr)
return False
for line in sorted(lines):
print(line, file=out)
if self.output != "-":
out.close()
else:
print(
f"Unspupported output format '{self.type}': No objects are written",
file=sys.stderr,
)
return False
print(
f"Saved: {len(picks):d} picks, {noAmps:d} amplitudes",
file=sys.stderr,
)
return True
def main(argv):
app = DumpPicks(len(argv), argv)
return app()
if __name__ == "__main__":
sys.exit(main(sys.argv))