1246 lines
40 KiB
Python
Executable File
1246 lines
40 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
###############################################################################
|
|
# Copyright (C) 2012 by gempa GmbH #
|
|
# #
|
|
# All Rights Reserved. #
|
|
# #
|
|
# NOTICE: All information contained herein is, and remains #
|
|
# the property of gempa GmbH and its suppliers, if any. The intellectual #
|
|
# and technical concepts contained herein are proprietary to gempa GmbH #
|
|
# and its suppliers. #
|
|
# Dissemination of this information or reproduction of this material #
|
|
# is strictly forbidden unless prior written permission is obtained #
|
|
# from gempa GmbH. #
|
|
# #
|
|
# Author: Enrico Ellguth, Stephan Herrnkind #
|
|
# Email: ellguth@gempa.de, herrnkind@gempa.de #
|
|
###############################################################################
|
|
|
|
"""Usage:
|
|
capstool [options] -P
|
|
capstool [options] -Q
|
|
capstool [options] [request-file(s)]
|
|
|
|
Retrieve data and meta information from CAPS server.
|
|
|
|
Options:
|
|
-h, --help Display this help message and exit.
|
|
-H, --host=HOST[:PORT] Host and optionally port of the CAPS server
|
|
(default: localhost:18002).
|
|
-s, --ssl Use secure socket layer (SSL).
|
|
-c, --credentials=USER[:PASS] Authentication credentials. If password is
|
|
omitted, it is asked for on command-line.
|
|
|
|
-P, --ping Retrieve server version information and exit.
|
|
|
|
-Q, Print availability extents of all data streams.
|
|
-I, --info-streams=FILTER Like -Q but with a use a regular filter
|
|
expression for the requested streams, e.g.,
|
|
AM.*.
|
|
--filter-list=FILTER Identical to -I.
|
|
--mtime [start]:[end] Restrict request to record modification time
|
|
window. Time format:
|
|
%Y,%m,%d[,%H[,%M[,%S[,%f]]]]
|
|
-X, --info-server Request server statistics in JSON format
|
|
--modified-after=TIME Limit server statistics request to data
|
|
modified after specific time. Time format:
|
|
%Y,%m,%d[,%H[,%M[,%S[,%f]]]]
|
|
--force Disable any confirmation prompts.
|
|
|
|
Options (request file, no data download):
|
|
-G, --print-gaps Request list of data gaps.
|
|
-S, --print-segments Request list of continuous data segments.
|
|
--tolerance=SECONDS Threshold in seconds defining a data gap (decimal
|
|
point, microsecond precision, default: 0).
|
|
-R, --resolution=DAYS The resolution in multiple of days of the returned
|
|
data segments or gaps (default: 0). A value of 0
|
|
returns segments based on stored data records. A
|
|
value larger than zero will return the minimum and
|
|
maximum data time of one, two or more days.
|
|
Consecutive segments will be merged if end and start
|
|
time are within the tolerance.
|
|
--print-stat Request storage information with a granularity of
|
|
one day.
|
|
--purge Deletes data from CAPS archive with a granularity of
|
|
one day. Any data file intersecting with the time
|
|
window will be purged. The user requires the purge
|
|
permission.
|
|
|
|
Options (request file and data download):
|
|
-o, --output-file=FILE Output file for received data (default: -).
|
|
The file name is used as a prefix with the
|
|
extension added based on the record type (MSEED,
|
|
RAW, ANY, META, HELI). Multiple files are created
|
|
if mixed data types are received. For 'ANY' records
|
|
the file name is set to PREFIX_STREAMID_DATE.TYPE
|
|
--any-date-format Date format to use for any files, see
|
|
'man strftime' (default: %Y%m%d_%H%M%S).
|
|
-t, --temp-file=FILE Use temporary file to store data. On success
|
|
move to output-file.
|
|
--rt Enable real-time mode.
|
|
--ooo, --out-of-order Request data in order of transmission time instead
|
|
of sampling time.
|
|
|
|
-D, --heli Request down-sampled data (1Hz). The server will
|
|
taper, bandpass filter and re-sample the data.
|
|
--itaper=SECONDS Timespan in SECONDS for the one-sided cosine taper.
|
|
--bandpass=RANGE Corner frequency RANGE of the bandpass filter,
|
|
e.g., 1.0:4.0.
|
|
|
|
-M, --meta Request record meta data only.
|
|
-v, --version=VERSION Request a specific format version. Currently only
|
|
supported in meta requests.
|
|
|
|
|
|
Request file format:
|
|
Line based with each line containing a start time, end time and a stream id:
|
|
- time representation: %Y,%m,%d[,%H[,%M[,%S[,%f]]]]
|
|
- leading zeros may be omitted.
|
|
- an open end time is expressed by an underscore (_)
|
|
- the stream ID is defined as NET STA LOC CHA with
|
|
- LOC omitted in case of an empty location code
|
|
- support for * and ? wildcards
|
|
Example:
|
|
2014,03,17,12,00,00 2014,03,17,13,00,00 AM R0F05 00 SHZ
|
|
2014,03,17,12,0,0 _ GE APE BH?
|
|
|
|
Examples:
|
|
Fetch data from CAPS server on localhost and save it to a file:
|
|
echo "2014,03,17,12,00,00 2014,03,17,13,00,00 NET STA LOC CHA" | capstool -o [file]
|
|
|
|
Fetch data from a CAPS server on HOSTNAME and save it to a file. Request details
|
|
are provided through the request file req.txt:
|
|
capstool -H HOSTNAME -o /tmp[file] req.txt
|
|
|
|
Fetch stream information from a CAPS server on HOSTNAME using SSL port and
|
|
authentication:
|
|
capstool -H HOSNAME:18004 -s -c USERNAME:PASSWORD -Q
|
|
|
|
Fetch gap statistic for given stream and time window from local CAPS server
|
|
echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool -G
|
|
|
|
Fetch segment statistic for given stream and time window from local CAPS server
|
|
using a gap threshold of 0.5 seconds
|
|
echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool -G --tolerance=0.5
|
|
|
|
Fetch disk usage statistic for given stream and time window from local CAPS server
|
|
echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool --print-stat
|
|
|
|
Purge data for given stream and time window from local CAPS server
|
|
echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool --purge
|
|
"""
|
|
|
|
import getopt
|
|
import getpass
|
|
import os
|
|
import re
|
|
import socket
|
|
import ssl
|
|
import struct
|
|
import sys
|
|
import traceback
|
|
|
|
from datetime import datetime, timedelta
|
|
from io import BytesIO
|
|
from typing import BinaryIO, List, NamedTuple, TextIO, Tuple
|
|
|
|
|
|
FullTimeFormat = "%F %T.%fZ"
|
|
|
|
CAPSTimeRegex = re.compile(
|
|
r"^\d{4},\d{1,2},\d{1,2}(,\d{1,2}(,\d{1,2}(,\d{1,2}(,\d{1,6})?)?)?)?$"
|
|
)
|
|
PIPE = "-"
|
|
|
|
ERR_OK = 0
|
|
ERR_UNKNOWN = 1
|
|
ERR_USAGE = 2
|
|
ERR_INPUT = 3
|
|
ERR_CONNECTION = 4
|
|
ERR_SERVER = 5
|
|
|
|
|
|
class CAPSToolError(Exception):
|
|
def __init__(self, message: str, error_code: int):
|
|
super().__init__(message)
|
|
self.error_code: int = error_code
|
|
|
|
|
|
class CAPSToolUsageError(CAPSToolError):
|
|
def __init__(self, message: str):
|
|
super().__init__(message, ERR_USAGE)
|
|
|
|
|
|
class CAPSToolInputError(CAPSToolError):
|
|
def __init__(self, message: str):
|
|
super().__init__(message, ERR_INPUT)
|
|
|
|
|
|
class CAPSToolConnectionError(CAPSToolError):
|
|
def __init__(self, message: str):
|
|
super().__init__(message, ERR_CONNECTION)
|
|
|
|
|
|
class CAPSToolServerError(CAPSToolError):
|
|
def __init__(self, message: str):
|
|
super().__init__(message, ERR_SERVER)
|
|
|
|
|
|
def py3bstr(s: str) -> bytes:
|
|
"""string to bytes"""
|
|
return s.encode("utf-8")
|
|
|
|
|
|
def py3ustr(b: bytes) -> str:
|
|
"""bytes to bytes"""
|
|
return b.decode("utf-8", "replace")
|
|
|
|
|
|
def error(msg: str) -> None:
|
|
print(f"[error] {msg}", file=sys.stderr)
|
|
|
|
|
|
def warning(msg: str) -> None:
|
|
print(f"[warning] {msg}", file=sys.stderr)
|
|
|
|
|
|
def info(msg: str) -> None:
|
|
print(f"[info] {msg}", file=sys.stderr)
|
|
|
|
|
|
def send(sock: socket.socket, data: str) -> None:
|
|
sock.send(py3bstr(data) + b"\n")
|
|
|
|
|
|
def read_buffer(sock: socket.socket, bufsize: int) -> bytearray:
|
|
data = bytearray()
|
|
|
|
while bufsize > 0:
|
|
req: int = min(1024, bufsize)
|
|
buf: bytes = sock.recv(req)
|
|
if len(buf) == 0:
|
|
break
|
|
data += buf
|
|
bufsize -= len(buf)
|
|
|
|
return data
|
|
|
|
|
|
def dataSize(fmt: str) -> Tuple[int, str]:
|
|
if fmt == "RAW/INT8":
|
|
return (1, "c")
|
|
|
|
if fmt == "RAW/INT16":
|
|
return (2, "h")
|
|
|
|
if fmt == "RAW/INT32":
|
|
return (4, "i")
|
|
|
|
if fmt == "RAW/INT64":
|
|
return (8, "q")
|
|
|
|
if fmt == "RAW/FLOAT":
|
|
return (4, "f")
|
|
|
|
if fmt == "RAW/DOUBLE":
|
|
return (8, "d")
|
|
|
|
return 0, ""
|
|
|
|
|
|
class Session(NamedTuple):
|
|
sid: str = ""
|
|
sfreq: str = ""
|
|
uom: str = ""
|
|
fmt: str = ""
|
|
|
|
def __bool__(self) -> bool:
|
|
return bool(self.sid)
|
|
|
|
|
|
def writeRAW(out: TextIO, buf: bytearray, session: Session) -> int:
|
|
"""
|
|
Write RAW record in SLIST format to output file
|
|
"""
|
|
|
|
startTime = unpackTime(buf).strftime("%Y-%m-%dT%H:%M:%S.%f")
|
|
size, t = dataSize(session.fmt)
|
|
if not size:
|
|
error(f"unsupported datatype: {session.fmt}")
|
|
return 0
|
|
|
|
headerLen = 12
|
|
dataLen = len(buf) - headerLen
|
|
sampleCount = dataLen // size
|
|
if sampleCount <= 0:
|
|
return 0
|
|
|
|
sid = session.sid.replace(".", "_")
|
|
dataType = session.fmt.split("/")[1]
|
|
|
|
toks = session.sfreq.split("/")
|
|
freq = int(toks[0]) / int(toks[1])
|
|
if freq.is_integer():
|
|
freq = int(freq)
|
|
|
|
# TIMESERIES CX_PB11__BHZ_R, 1588 samples, 40 sps, 2019-05-19T19:10:53.225000,
|
|
# SLIST, FLOAT, M/S
|
|
print(
|
|
f"TIMESERIES {sid}_R, {sampleCount} samples, {freq} sps, {startTime}, SLIST, "
|
|
f"{dataType}, {session.uom}",
|
|
file=out,
|
|
)
|
|
|
|
for i in range(sampleCount):
|
|
ofs = i * size + headerLen
|
|
value = struct.unpack(t, buf[ofs : ofs + size])[0]
|
|
print(value, file=out)
|
|
|
|
return sampleCount
|
|
|
|
|
|
def writeHeli(out: TextIO, buf: bytearray, session: Session) -> int:
|
|
"""
|
|
Write HELI in SLIST format to output file
|
|
Each HELI sample consists of 3 double values:
|
|
start time (seconds since epoch), minimum, maximum
|
|
The expected sample rate is 1s. We double the rate and write the minimum and
|
|
aximum value as alternating values.
|
|
"""
|
|
|
|
recLen = 3 * 8
|
|
bufLen = len(buf)
|
|
sampleCount = bufLen // recLen
|
|
if sampleCount <= 0:
|
|
return 0
|
|
|
|
secs, minimum, maximum = struct.unpack("<ddd", buf[:recLen])
|
|
startTime = datetime.utcfromtimestamp(secs).strftime("%Y-%m-%dT%H:%M:%S.%f")
|
|
sid = session.sid.replace(".", "_")
|
|
|
|
# original frequency unused
|
|
# toks = session.sfreq.split("/")
|
|
# freq = int(toks[0]) / int(toks[1])
|
|
|
|
# TIMESERIES CX_PB11__BHZ_R, 12 samples, 2 sps, 2019-05-19T19:10:53.225000, SLIST,
|
|
# FLOAT, cnt
|
|
print(
|
|
f"TIMESERIES {sid}_R, {sampleCount * 2} samples, 2 sps, {startTime}, SLIST, "
|
|
f"FLOAT, {session.uom}",
|
|
file=out,
|
|
)
|
|
print(minimum, file=out)
|
|
print(maximum, file=out)
|
|
|
|
for i in range(1, sampleCount):
|
|
ofs = i * recLen
|
|
_secs, minimum, maximum = struct.unpack("<ddd", buf[ofs : ofs + recLen])
|
|
print(minimum, file=out)
|
|
print(maximum, file=out)
|
|
|
|
return sampleCount
|
|
|
|
|
|
def unpackTime(buf: bytearray, offset: int = 0) -> datetime:
|
|
secs, usecs = struct.unpack("<QI", buf[offset : offset + 12])
|
|
return datetime.utcfromtimestamp(secs + usecs / 10**6)
|
|
|
|
|
|
class SessionTable:
|
|
def __init__(self) -> None:
|
|
self.session_by_id: dict[int, Session] = {}
|
|
self.id_by_sid: dict[str, int] = {}
|
|
|
|
def _handleRequests(self, lines: List[str]) -> None:
|
|
d: dict[str, str] = {}
|
|
|
|
for line in lines:
|
|
if line == "END":
|
|
break
|
|
|
|
start = 0
|
|
pos = 0
|
|
while True:
|
|
pos = line.find(":", start)
|
|
if pos == -1:
|
|
break
|
|
|
|
key = line[start:pos]
|
|
|
|
start = pos + 1
|
|
pos = line.find(",", start)
|
|
if pos == -1:
|
|
value = line[start:]
|
|
else:
|
|
value = line[start:pos]
|
|
start = pos + 1
|
|
|
|
d[key] = value
|
|
|
|
if "ID" in d:
|
|
sessionID = int(d["ID"])
|
|
sid = d["SID"]
|
|
if sessionID == -1:
|
|
sessionID = self.id_by_sid[sid]
|
|
del self.id_by_sid[sid]
|
|
del self.session_by_id[sessionID]
|
|
else:
|
|
session = Session(sid, d["SFREQ"], d["UOM"], d["FMT"])
|
|
|
|
self.id_by_sid[sid] = sessionID
|
|
self.session_by_id[sessionID] = session
|
|
|
|
def handleResponse(self, sock: socket.socket) -> Tuple[bytearray, Session]:
|
|
# read initial response from server
|
|
buf = read_buffer(sock, 6)
|
|
if len(buf) != 6:
|
|
raise CAPSToolServerError("invalid response from server")
|
|
|
|
(sessionID, size) = struct.unpack("=HI", buf)
|
|
buf = read_buffer(sock, size)
|
|
|
|
if sessionID == 0:
|
|
resp = py3ustr(buf)
|
|
if resp.startswith("ERROR:"):
|
|
raise CAPSToolServerError(f"server responded with: {resp.strip()}")
|
|
|
|
toks = resp.split("\n")
|
|
|
|
# end of data
|
|
if toks[0] == "EOD":
|
|
return bytearray(), Session()
|
|
|
|
# session table modified
|
|
if toks[0] == "REQUESTS":
|
|
self._handleRequests(toks[1:])
|
|
elif toks[0] != "STATUS OK":
|
|
warning(f"server responded with unknown key word: {toks[0]}")
|
|
|
|
return buf, Session()
|
|
|
|
# unknown sessionID
|
|
if sessionID not in self.session_by_id:
|
|
raise CAPSToolServerError(
|
|
f"server responded with unknown session id: {sessionID}"
|
|
)
|
|
|
|
# read packet data
|
|
return buf, self.session_by_id[sessionID]
|
|
|
|
|
|
class CAPSTool:
|
|
class SEEDParams:
|
|
def __init__(self) -> None:
|
|
self.enable = False
|
|
self.resp_dict = False
|
|
|
|
def __init__(self) -> None:
|
|
self.host = "localhost"
|
|
self.port = 18002
|
|
self.useSSL = False
|
|
|
|
self.username = ""
|
|
self.password = ""
|
|
|
|
self.pingServer = False
|
|
self.infoStreams = ""
|
|
self.infoServer = False
|
|
self.mtime = ""
|
|
|
|
self.inputFiles = [PIPE]
|
|
self.output = PIPE
|
|
self.anyDateFormat = "%Y%m%d_%H%M%S"
|
|
self.tmpOutputFile = ""
|
|
|
|
self.printGaps = False
|
|
self.printSegments = False
|
|
self.printStat = False
|
|
self.tolerance = 0
|
|
self.resolution = 0
|
|
|
|
self.real_time = False
|
|
self.ooo = False
|
|
|
|
self.heli = False
|
|
self.itaper = ""
|
|
self.bandpass = ""
|
|
|
|
self.meta = False
|
|
self.formatVersion = ""
|
|
|
|
self.purge = False
|
|
self.force = False
|
|
|
|
self.SEED = self.SEEDParams()
|
|
|
|
self._stdin_dirty = False
|
|
|
|
def parse_args(self, args: List[str]) -> None:
|
|
try:
|
|
opts, args = getopt.gnu_getopt(
|
|
args,
|
|
"H:sc:PQI:XGSR:o:t:MDv:",
|
|
[
|
|
"ping",
|
|
"host=",
|
|
"ssl",
|
|
"credentials=",
|
|
"ping",
|
|
"info-streams=",
|
|
"filter-list=",
|
|
"mtime=",
|
|
"info-server",
|
|
"modified-after=",
|
|
"print-gaps",
|
|
"print-segments",
|
|
"output-file=",
|
|
"any-date-format=",
|
|
"temp-file=",
|
|
"rt",
|
|
"tolerance=",
|
|
"resolution=",
|
|
"ooo",
|
|
"out-of-order",
|
|
"heli",
|
|
"itaper=",
|
|
"bandpass=",
|
|
"meta",
|
|
"version=",
|
|
"purge",
|
|
"force",
|
|
"print-stat",
|
|
],
|
|
)
|
|
except getopt.GetoptError as err:
|
|
# will print something like "option -a not recognized"
|
|
raise CAPSToolUsageError(str(err)) from err
|
|
|
|
infoCommands = set()
|
|
downloadOpts = set()
|
|
|
|
for o, a in opts:
|
|
if o in ["-H", "--host"]:
|
|
toks = a.split(":", 1)
|
|
if toks[0]:
|
|
self.host = toks[0]
|
|
if len(toks) == 2 and toks[1]:
|
|
try:
|
|
self.port = int(toks[1])
|
|
except ValueError as e:
|
|
raise CAPSToolUsageError(
|
|
f"invalid port given: {toks[1]}"
|
|
) from e
|
|
|
|
elif o in ["-s", "--ssl"]:
|
|
self.useSSL = True
|
|
|
|
elif o in ["-c", "--credentials"]:
|
|
toks = a.split(":", 1)
|
|
if len(toks) > 1:
|
|
self.username = toks[0]
|
|
self.password = toks[1]
|
|
else:
|
|
self.username = toks[0]
|
|
# ask for password
|
|
self.password = getpass.getpass()
|
|
|
|
elif o in ["-P", "--ping"]:
|
|
self.pingServer = True
|
|
infoCommands.add("PING")
|
|
|
|
elif o in ["-Q"]:
|
|
self.infoStreams = "*"
|
|
infoCommands.add("INFO STREAMS")
|
|
|
|
elif o in ["-I", "--info-streams", "--filter-list"]:
|
|
self.infoStreams = a if a else "*"
|
|
infoCommands.add("INFO STREAMS")
|
|
|
|
elif o in ["--mtime"]:
|
|
toks = a.split(":")
|
|
if len(toks) != 2:
|
|
raise CAPSToolUsageError(
|
|
"invalid numbers of colons in --mtime parameter"
|
|
)
|
|
|
|
if toks[0] and not CAPSTimeRegex.match(toks[0]):
|
|
raise CAPSToolUsageError("invalid start time in --mtime parameter")
|
|
|
|
if toks[1] and not CAPSTimeRegex.match(toks[1]):
|
|
raise CAPSToolUsageError("invalid end time in --mtime parameter")
|
|
|
|
self.mtime = a
|
|
|
|
elif o in ["-X", "--info-server"]:
|
|
infoCommands.add("INFO SERVER")
|
|
|
|
elif o in ["--modified-after"]:
|
|
if not CAPSTimeRegex.match(a):
|
|
raise CAPSToolUsageError("invalid time in --modified-after")
|
|
|
|
self.mtime = a
|
|
|
|
elif o in ["-G", "--print-gaps"]:
|
|
self.printGaps = True
|
|
infoCommands.add("GAPS")
|
|
|
|
elif o in ["-S", "--print-segments"]:
|
|
self.printSegments = True
|
|
infoCommands.add("SEGMENTS")
|
|
|
|
elif o in ["--print-stat"]:
|
|
self.printStat = True
|
|
infoCommands.add("STATS")
|
|
|
|
elif o in ["--tolerance"]:
|
|
try:
|
|
self.tolerance = int(round(float(a) * 10**6))
|
|
except ValueError as e:
|
|
raise CAPSToolUsageError(
|
|
"invalid floating point value in --tolerance parameter"
|
|
) from e
|
|
|
|
elif o in ["-R", "--resolution"]:
|
|
if not a.isdigit():
|
|
raise CAPSToolUsageError(
|
|
"invalid integer value in --resolution parameter"
|
|
)
|
|
|
|
self.resolution = int(a)
|
|
|
|
elif o in ["-o", "--output-file"]:
|
|
self.output = a
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["--any-date-format"]:
|
|
self.anyDateFormat = a
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["-t", "--temp-file"]:
|
|
self.tmpOutputFile = a
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["--rt"]:
|
|
self.real_time = True
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["--ooo", "--out-of-order"]:
|
|
self.ooo = True
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["-D", "--heli"]:
|
|
self.heli = True
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["--itaper"]:
|
|
self.itaper = a
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["--bandpass"]:
|
|
self.bandpass = a
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["-M", "--meta"]:
|
|
self.meta = True
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["-v", "--version"]:
|
|
self.formatVersion = a
|
|
downloadOpts.add(o)
|
|
|
|
elif o in ["--purge"]:
|
|
self.purge = True
|
|
|
|
elif o in ["--force"]:
|
|
self.force = True
|
|
|
|
else:
|
|
raise CAPSToolUsageError(f"unhandled option: {o}")
|
|
|
|
# check for invalid command combinations
|
|
if len(infoCommands) > 1:
|
|
raise CAPSToolUsageError(
|
|
f"more than one info command requested: {', '.join(infoCommands)}"
|
|
)
|
|
|
|
# read request file arguments
|
|
if len(args) > 0:
|
|
self.inputFiles = args
|
|
if self.pingServer:
|
|
raise CAPSToolUsageError(
|
|
"PING command and request file(s) may not be combined"
|
|
)
|
|
|
|
if self.infoStreams:
|
|
raise CAPSToolUsageError(
|
|
"INFO STREAMS command and request file(s) may not be combined"
|
|
)
|
|
|
|
# check for info command and download option combinations
|
|
if infoCommands and downloadOpts:
|
|
raise CAPSToolUsageError(
|
|
"info commands and download options may not be combined\n"
|
|
f" info command(s) : {', '.join(infoCommands)}\n"
|
|
f" download option(s): {', '.join(downloadOpts)}"
|
|
)
|
|
|
|
# check for invalid parameter combinations
|
|
if self.tmpOutputFile and self.output is PIPE:
|
|
raise CAPSToolUsageError(
|
|
"temporary output file not supported when writing to stdout"
|
|
)
|
|
|
|
if self.tolerance and not (self.printGaps or self.printSegments):
|
|
raise CAPSToolUsageError(
|
|
"tolerance is only supported in GAPS and SEGMENTS requests"
|
|
)
|
|
|
|
if self.resolution and not (self.printGaps or self.printSegments):
|
|
raise CAPSToolUsageError(
|
|
"resolution is only supported in GAPS and SEGMENTS requests"
|
|
)
|
|
|
|
if self.formatVersion and not self.meta:
|
|
raise CAPSToolUsageError(
|
|
"format version is currently only supported in META requests"
|
|
)
|
|
|
|
if self.heli:
|
|
if self.meta:
|
|
raise CAPSToolUsageError("HELI and META requests may not be combined")
|
|
|
|
elif self.itaper:
|
|
raise CAPSToolUsageError(
|
|
"itaper parameter only supported for HELI requests"
|
|
)
|
|
|
|
elif self.bandpass:
|
|
raise CAPSToolUsageError(
|
|
"bandpass parameter only supported for HELI requests"
|
|
)
|
|
|
|
if self.mtime and self.real_time:
|
|
raise CAPSToolUsageError(
|
|
"mtime data filter not available in real-time mode"
|
|
)
|
|
|
|
def connect(self) -> socket.socket:
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
try:
|
|
if self.useSSL:
|
|
context = ssl.SSLContext()
|
|
sock = context.wrap_socket(sock)
|
|
sock.connect((self.host, self.port))
|
|
except socket.error as e:
|
|
raise CAPSToolConnectionError(
|
|
f"could not establish connection to {self.host}:{self.port}"
|
|
) from e
|
|
|
|
info(f"connected to {self.host}:{self.port}")
|
|
|
|
return sock
|
|
|
|
@staticmethod
|
|
def disconnect(sock: socket.socket) -> None:
|
|
send(sock, "BYE")
|
|
sock.close()
|
|
|
|
def authenticate(self, sock: socket.socket) -> None:
|
|
if self.username:
|
|
send(sock, f"AUTH {self.username} {self.password}")
|
|
|
|
def beginRequest(self, sock: socket.socket) -> None:
|
|
send(sock, "BEGIN REQUEST")
|
|
|
|
if self.mtime:
|
|
send(sock, f"MTIME {self.mtime}")
|
|
|
|
if self.printGaps:
|
|
send(sock, "GAPS ON")
|
|
|
|
if self.printSegments:
|
|
send(sock, "SEGMENTS ON")
|
|
|
|
if self.printStat:
|
|
send(sock, "STAT")
|
|
|
|
if self.tolerance:
|
|
send(sock, f"TOLERANCE {self.tolerance}")
|
|
|
|
if self.resolution:
|
|
send(sock, f"RESOLUTION {self.resolution}")
|
|
|
|
if not self.real_time:
|
|
send(sock, "REALTIME OFF")
|
|
|
|
if self.ooo:
|
|
send(sock, "OUTOFORDER ON")
|
|
|
|
if self.meta:
|
|
if self.formatVersion:
|
|
send(sock, f"META@{self.formatVersion} ON")
|
|
else:
|
|
send(sock, "META ON")
|
|
|
|
@staticmethod
|
|
def endRequest(sock: socket.socket) -> None:
|
|
send(sock, "END")
|
|
|
|
@staticmethod
|
|
def readDefaultResponse(sock: socket.socket) -> None:
|
|
buf = read_buffer(sock, 6)
|
|
if len(buf) != 6:
|
|
raise CAPSToolServerError("unexpected response from server (len != 6)")
|
|
|
|
(code, size) = struct.unpack("=HI", buf)
|
|
if code != 0:
|
|
raise CAPSToolServerError("unexpected response from server (id != 0)")
|
|
|
|
if size < 0:
|
|
raise CAPSToolServerError(
|
|
"unexpected response from server (negative data size)"
|
|
)
|
|
|
|
buf = read_buffer(sock, size)
|
|
print(py3ustr(buf))
|
|
|
|
def addRequestLinesFromFile(self, buffer: BinaryIO, f: TextIO) -> None:
|
|
def handleError(reason: str) -> None:
|
|
msg = f"invalid request line #{lc}, {reason}"
|
|
if f.isatty():
|
|
error(msg)
|
|
else:
|
|
print(f" > {line}", file=sys.stderr)
|
|
raise CAPSToolInputError(msg)
|
|
|
|
if self.heli:
|
|
heli_filter = ""
|
|
if self.itaper:
|
|
heli_filter += f" ITAPER {self.itaper}"
|
|
if self.bandpass:
|
|
heli_filter += f" BANDPASS {self.bandpass}"
|
|
|
|
lc = 0
|
|
for line in f:
|
|
lc += 1
|
|
line = line.strip()
|
|
if not line or line[0] == "#":
|
|
print(f" + {lc:>5}: SKIP", file=sys.stderr)
|
|
continue
|
|
|
|
toks = line.split()
|
|
if len(toks) < 5:
|
|
handleError("less than 5 columns given")
|
|
continue
|
|
|
|
if toks[0] == "_":
|
|
startTime = ""
|
|
else:
|
|
if CAPSTimeRegex.match(toks[0]):
|
|
startTime = toks[0]
|
|
else:
|
|
handleError("invalid start time in column 1")
|
|
continue
|
|
|
|
if toks[1] == "_":
|
|
endTime = ""
|
|
else:
|
|
if CAPSTimeRegex.match(toks[1]):
|
|
endTime = toks[1]
|
|
else:
|
|
handleError("invalid end time in column 2")
|
|
continue
|
|
|
|
print(f" + {lc:>5}: OK", file=sys.stderr)
|
|
|
|
# pylint: disable=consider-using-f-string
|
|
if len(toks) == 5:
|
|
streamID = "{}.{}..{}".format(*toks[2:5])
|
|
else:
|
|
streamID = "{}.{}.{}.{}".format(*toks[2:6])
|
|
|
|
if self.heli:
|
|
buffer.write(py3bstr(f"HELI ADD {streamID}{heli_filter}\n"))
|
|
else:
|
|
buffer.write(py3bstr(f"STREAM ADD {streamID}\n"))
|
|
buffer.write(py3bstr(f"TIME {startTime}:{endTime}\n"))
|
|
|
|
def prompt_for_confirmation(self, text: str) -> bool:
|
|
if self._stdin_dirty:
|
|
os.close(0)
|
|
tty = "/dev/tty"
|
|
if os.open(tty, os.O_RDONLY):
|
|
raise OSError(f"could not open {tty}")
|
|
|
|
self._stdin_dirty = False
|
|
|
|
while True:
|
|
response = input(text).lower()
|
|
if response in ["yes", "y"]:
|
|
return True
|
|
if response in ["no", "n"]:
|
|
return False
|
|
|
|
print(" Respond with 'y' or 'n'.")
|
|
|
|
def requestLines(self) -> BytesIO:
|
|
buffer = BytesIO()
|
|
|
|
# read request lines from stdin or files(s)
|
|
if PIPE in self.inputFiles:
|
|
if sys.stdin.isatty():
|
|
print(
|
|
"""Please input request lines of FORMAT followed by Ctrl+D
|
|
FORMAT: starttime endtime NET STA [LOC] CHA
|
|
Example: 2022,05,01,12,00,00 2022,05,03,00,00,00 AM R0F05 * *""",
|
|
file=sys.stderr,
|
|
)
|
|
else:
|
|
info("reading request lines from stdin:")
|
|
self.addRequestLinesFromFile(buffer, sys.stdin)
|
|
self._stdin_dirty = True
|
|
else:
|
|
for file in self.inputFiles:
|
|
info(f"reading request lines from {file}:")
|
|
try:
|
|
with open(file, "r", encoding="utf8") as f:
|
|
self.addRequestLinesFromFile(buffer, f)
|
|
except IOError as e:
|
|
raise CAPSToolInputError(
|
|
f"could not read request file {file}"
|
|
) from e
|
|
|
|
return buffer
|
|
|
|
def cmdPing(self) -> None:
|
|
sock = self.connect()
|
|
|
|
send(sock, "HELLO")
|
|
|
|
self.readDefaultResponse(sock)
|
|
self.disconnect(sock)
|
|
|
|
def cmdInfoStreams(self) -> None:
|
|
sock = self.connect()
|
|
self.authenticate(sock)
|
|
|
|
send(sock, f"INFO STREAMS {self.infoStreams}")
|
|
|
|
self.readDefaultResponse(sock)
|
|
self.disconnect(sock)
|
|
|
|
def cmdInfoServer(self) -> None:
|
|
sock = self.connect()
|
|
self.authenticate(sock)
|
|
|
|
mtime = f" MODIFIED AFTER {self.mtime}" if self.mtime else ""
|
|
send(sock, f"INFO SERVER{mtime}")
|
|
|
|
self.readDefaultResponse(sock)
|
|
self.disconnect(sock)
|
|
|
|
def cmdSegments(self) -> None:
|
|
buffer = self.requestLines()
|
|
|
|
sock = self.connect()
|
|
self.authenticate(sock)
|
|
self.beginRequest(sock)
|
|
sock.send(buffer.getvalue())
|
|
self.endRequest(sock)
|
|
|
|
# pylint: disable=consider-using-f-string
|
|
print(f"{'Stream ID': <15} {'Start': <27} {'End': <27}")
|
|
sessionTable = SessionTable()
|
|
while True:
|
|
buf, session = sessionTable.handleResponse(sock)
|
|
|
|
if not buf: # EOD
|
|
break
|
|
|
|
if not session:
|
|
continue
|
|
|
|
start = unpackTime(buf)
|
|
end = unpackTime(buf, 12)
|
|
|
|
print(
|
|
"{0: <15} {1: <27} {2: <27}".format(
|
|
session.sid,
|
|
start.strftime(FullTimeFormat),
|
|
end.strftime(FullTimeFormat),
|
|
)
|
|
)
|
|
|
|
def cmdStat(self) -> None:
|
|
buffer = self.requestLines()
|
|
|
|
sock = self.connect()
|
|
self.authenticate(sock)
|
|
self.beginRequest(sock)
|
|
sock.send(buffer.getvalue())
|
|
self.endRequest(sock)
|
|
|
|
self.readDefaultResponse(sock)
|
|
self.disconnect(sock)
|
|
|
|
def cmdPurge(self) -> None:
|
|
buffer = self.requestLines()
|
|
|
|
if not self.force and not self.prompt_for_confirmation(
|
|
"Do you really want to remove the selected streams permanently from CAPS "
|
|
"archive (y/n)? "
|
|
):
|
|
return
|
|
|
|
sock = self.connect()
|
|
self.authenticate(sock)
|
|
|
|
send(sock, "BEGIN PURGE")
|
|
sock.send(buffer.getvalue())
|
|
self.endRequest(sock)
|
|
|
|
self.readDefaultResponse(sock)
|
|
self.disconnect(sock)
|
|
|
|
def cmdDownload(self) -> None:
|
|
# pylint: disable=consider-using-with
|
|
# initialize out variables which depend on Python version used
|
|
mseed_out = any_out = raw_out = meta_out = heli_out = None
|
|
if not self.output or self.output == PIPE:
|
|
write_to_stdout = True
|
|
mseed_out = any_out = sys.stdout.buffer
|
|
raw_out = meta_out = heli_out = sys.stdout
|
|
mseed_file_name = any_file_name = raw_file_name = meta_file_name = (
|
|
heli_file_name
|
|
) = "stdout"
|
|
else:
|
|
write_to_stdout = False
|
|
if self.output[-6:].lower() == ".mseed":
|
|
mseed_file_name = self.output
|
|
else:
|
|
mseed_file_name = self.output + ".mseed"
|
|
|
|
any_time = f"_{self.anyDateFormat}" if self.anyDateFormat else ""
|
|
any_file_name = f"files of form {self.output}_STREAMID{any_time}.TYPE"
|
|
|
|
if self.output[-4:].lower() == ".raw":
|
|
raw_file_name = self.output
|
|
else:
|
|
raw_file_name = self.output + ".raw"
|
|
|
|
if self.output[-5:].lower() == ".meta":
|
|
meta_file_name = self.output
|
|
else:
|
|
meta_file_name = self.output + ".meta"
|
|
|
|
if self.output[-5:].lower() == ".heli":
|
|
heli_file_name = self.output
|
|
else:
|
|
heli_file_name = self.output + ".heli"
|
|
|
|
mseed_bytes = 0
|
|
any_bytes = 0
|
|
raw_samples = 0
|
|
heli_samples = 0
|
|
meta_records = 0
|
|
|
|
buffer = self.requestLines()
|
|
|
|
sock = self.connect()
|
|
self.authenticate(sock)
|
|
self.beginRequest(sock)
|
|
sock.send(buffer.getvalue())
|
|
self.endRequest(sock)
|
|
|
|
sessionTable = SessionTable()
|
|
|
|
while True:
|
|
buf, session = sessionTable.handleResponse(sock)
|
|
if not buf: # EOD
|
|
break
|
|
|
|
if not session:
|
|
continue
|
|
|
|
# MSEED: Write binary MSeed data to stdout or single output file
|
|
if session.fmt == "MSEED":
|
|
if not mseed_out:
|
|
mseed_out = open(mseed_file_name, "wb")
|
|
|
|
mseed_out.write(buf)
|
|
mseed_bytes += len(buf)
|
|
|
|
# ANY: Write binary data to stdout or individual output files
|
|
elif session.fmt == "ANY":
|
|
if any_out:
|
|
any_out.write(buf)
|
|
any_bytes += len(buf)
|
|
continue
|
|
|
|
type_len = 4
|
|
for i in range(0, 4):
|
|
if buf[i] == 0:
|
|
type_len = i
|
|
break
|
|
packetType = py3ustr(buf[0:type_len]).lower()
|
|
|
|
year, yday, hour, minute, second, usec = struct.unpack(
|
|
"=hHBBBI", buf[5:16]
|
|
)
|
|
|
|
filename = f"{self.output}_{session.sid}"
|
|
if self.anyDateFormat:
|
|
time = datetime(year, 1, 1, hour, minute, second, usec)
|
|
time += timedelta(days=yday)
|
|
filename += f"_{time.strftime(self.anyDateFormat)}"
|
|
|
|
ext = f".{packetType if packetType else 'any'}"
|
|
if filename[-len(ext) :].lower() != ext:
|
|
filename += ext
|
|
|
|
if self.tmpOutputFile:
|
|
with open(self.tmpOutputFile, "wb") as f:
|
|
f.write(buf[31:])
|
|
os.rename(self.tmpOutputFile, filename)
|
|
|
|
else:
|
|
with open(filename, "wb") as f:
|
|
f.write(buf[31:])
|
|
|
|
any_bytes += len(buf) - 31
|
|
|
|
# RAW: Write SLIST string data to stdout or single output file
|
|
elif session.fmt[:3] == "RAW":
|
|
if not raw_out:
|
|
raw_out = open(raw_file_name, "w", encoding="utf8")
|
|
|
|
raw_samples += writeRAW(raw_out, buf, session)
|
|
|
|
# HELI: Write SLIST string data to stdout or single output file
|
|
elif session.fmt[:4] == "HELI":
|
|
if not heli_out:
|
|
heli_out = open(heli_file_name, "w", encoding="utf8")
|
|
|
|
heli_samples = writeHeli(heli_out, buf, session)
|
|
|
|
# META: Write string data to stdout or single output file
|
|
elif session.fmt == "META" or session.fmt[:5] == "META@":
|
|
version = 1
|
|
if len(session.fmt) > 4:
|
|
try:
|
|
version = int(session.fmt[5:])
|
|
except ValueError:
|
|
error(
|
|
f"Invalid META record version number '{session.fmt[5:]}' "
|
|
f"for SID '{session.sid}'"
|
|
)
|
|
continue
|
|
|
|
if version > 2:
|
|
error(
|
|
f"Unsupported META record version number '{version}' for SID "
|
|
f"'{session.sid}'"
|
|
)
|
|
continue
|
|
|
|
if version > 1:
|
|
arrivalTime = unpackTime(buf, 24).strftime(FullTimeFormat)
|
|
else:
|
|
arrivalTime = ""
|
|
|
|
if not meta_out:
|
|
meta_out = open(meta_file_name, "w", encoding="utf8")
|
|
|
|
# print header for first record
|
|
if not meta_records:
|
|
print(
|
|
"#Stream ID|Sampling Frequency|Unit of Measurement"
|
|
"|Start Time|End Time|Arrival Time",
|
|
file=meta_out,
|
|
)
|
|
|
|
print(
|
|
f"{session.sid}|{session.sfreq}|{session.uom}|"
|
|
f"{unpackTime(buf, 0).strftime(FullTimeFormat)}|"
|
|
f"{unpackTime(buf, 12).strftime(FullTimeFormat)}|{arrivalTime}",
|
|
file=meta_out,
|
|
)
|
|
meta_records += 1
|
|
|
|
# unsupported session format
|
|
else:
|
|
error(f"Unsupported session FMT '{session.fmt}' in SID '{session.sid}'")
|
|
|
|
# cleanup: flush buffers, close files
|
|
if write_to_stdout:
|
|
sys.stdout.flush()
|
|
else:
|
|
for fd in [mseed_out, raw_out, meta_out, heli_out]:
|
|
if fd:
|
|
fd.close()
|
|
|
|
if any((mseed_bytes, any_bytes, raw_samples, heli_samples, meta_records)):
|
|
msg = "Data acquisition completed:"
|
|
if mseed_bytes:
|
|
msg += f"\n + MSEED: Wrote {mseed_bytes} bytes to {mseed_file_name}"
|
|
if any_bytes:
|
|
msg += f"\n + ANY : Wrote {any_bytes} bytes to {any_file_name}"
|
|
if raw_samples:
|
|
msg += f"\n + RAW : Wrote {raw_samples} samples to {raw_file_name}"
|
|
if heli_samples:
|
|
msg += f"\n + HELI : Wrote {heli_samples} samples to {heli_file_name}"
|
|
if meta_records:
|
|
msg += f"\n + META : Wrote {meta_records} records to {meta_file_name}"
|
|
info(msg)
|
|
|
|
else:
|
|
info("No data written. Check request.")
|
|
|
|
self.disconnect(sock)
|
|
|
|
def __call__(self) -> None:
|
|
if self.pingServer:
|
|
self.cmdPing()
|
|
elif self.infoStreams:
|
|
self.cmdInfoStreams()
|
|
elif self.infoServer:
|
|
self.cmdInfoServer()
|
|
elif self.printSegments or self.printGaps:
|
|
self.cmdSegments()
|
|
elif self.printStat:
|
|
self.cmdStat()
|
|
elif self.purge:
|
|
self.cmdPurge()
|
|
else:
|
|
self.cmdDownload()
|
|
|
|
|
|
def main(args: List[str]) -> None:
|
|
if "-h" in args or "--help" in args:
|
|
print(__doc__, file=sys.stderr)
|
|
return
|
|
|
|
app = CAPSTool()
|
|
app.parse_args(args)
|
|
app()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
exit_code = ERR_OK
|
|
try:
|
|
main(sys.argv[1:])
|
|
except KeyboardInterrupt:
|
|
error("interrupted")
|
|
except EOFError:
|
|
error("end of file")
|
|
except CAPSToolError as main_e:
|
|
error(str(main_e))
|
|
exit_code = main_e.error_code
|
|
except Exception as main_e:
|
|
error(str(main_e))
|
|
traceback.print_exc(file=sys.stderr)
|
|
exit_code = ERR_UNKNOWN
|
|
|
|
sys.exit(exit_code)
|