#!/usr/bin/env python3 ############################################################################### # Copyright (C) 2012 by gempa GmbH # # # # All Rights Reserved. # # # # NOTICE: All information contained herein is, and remains # # the property of gempa GmbH and its suppliers, if any. The intellectual # # and technical concepts contained herein are proprietary to gempa GmbH # # and its suppliers. # # Dissemination of this information or reproduction of this material # # is strictly forbidden unless prior written permission is obtained # # from gempa GmbH. # # # # Author: Enrico Ellguth, Stephan Herrnkind # # Email: ellguth@gempa.de, herrnkind@gempa.de # ############################################################################### """Usage: capstool [options] -P capstool [options] -Q capstool [options] [request-file(s)] Retrieve data and meta information from CAPS server. Options: -h, --help Display this help message and exit. -H, --host=HOST[:PORT] Host and optionally port of the CAPS server (default: localhost:18002). -s, --ssl Use secure socket layer (SSL). -c, --credentials=USER[:PASS] Authentication credentials. If password is omitted, it is asked for on command-line. -P, --ping Retrieve server version information and exit. -Q, Print availability extents of all data streams. -I, --info-streams=FILTER Like -Q but with a use a regular filter expression for the requested streams, e.g., AM.*. --filter-list=FILTER Identical to -I. --mtime [start]:[end] Restrict request to record modification time window. Time format: %Y,%m,%d[,%H[,%M[,%S[,%f]]]] -X, --info-server Request server statistics in JSON format --modified-after=TIME Limit server statistics request to data modified after specific time. Time format: %Y,%m,%d[,%H[,%M[,%S[,%f]]]] --force Disable any confirmation prompts. Options (request file, no data download): -G, --print-gaps Request list of data gaps. -S, --print-segments Request list of continuous data segments. --tolerance=SECONDS Threshold in seconds defining a data gap (decimal point, microsecond precision, default: 0). -R, --resolution=DAYS The resolution in multiple of days of the returned data segments or gaps (default: 0). A value of 0 returns segments based on stored data records. A value larger than zero will return the minimum and maximum data time of one, two or more days. Consecutive segments will be merged if end and start time are within the tolerance. --print-stat Request storage information with a granularity of one day. --purge Deletes data from CAPS archive with a granularity of one day. Any data file intersecting with the time window will be purged. The user requires the purge permission. Options (request file and data download): -o, --output-file=FILE Output file for received data (default: -). The file name is used as a prefix with the extension added based on the record type (MSEED, RAW, ANY, META, HELI). Multiple files are created if mixed data types are received. For 'ANY' records the file name is set to PREFIX_STREAMID_DATE.TYPE --any-date-format Date format to use for any files, see 'man strftime' (default: %Y%m%d_%H%M%S). -t, --temp-file=FILE Use temporary file to store data. On success move to output-file. --rt Enable real-time mode. --ooo, --out-of-order Request data in order of transmission time instead of sampling time. -D, --heli Request down-sampled data (1Hz). The server will taper, bandpass filter and re-sample the data. --itaper=SECONDS Timespan in SECONDS for the one-sided cosine taper. --bandpass=RANGE Corner frequency RANGE of the bandpass filter, e.g., 1.0:4.0. -M, --meta Request record meta data only. -v, --version=VERSION Request a specific format version. Currently only supported in meta requests. Request file format: Line based with each line containing a start time, end time and a stream id: - time representation: %Y,%m,%d[,%H[,%M[,%S[,%f]]]] - leading zeros may be omitted. - an open end time is expressed by an underscore (_) - the stream ID is defined as NET STA LOC CHA with - LOC omitted in case of an empty location code - support for * and ? wildcards Example: 2014,03,17,12,00,00 2014,03,17,13,00,00 AM R0F05 00 SHZ 2014,03,17,12,0,0 _ GE APE BH? Examples: Fetch data from CAPS server on localhost and save it to a file: echo "2014,03,17,12,00,00 2014,03,17,13,00,00 NET STA LOC CHA" | capstool -o [file] Fetch data from a CAPS server on HOSTNAME and save it to a file. Request details are provided through the request file req.txt: capstool -H HOSTNAME -o /tmp[file] req.txt Fetch stream information from a CAPS server on HOSTNAME using SSL port and authentication: capstool -H HOSNAME:18004 -s -c USERNAME:PASSWORD -Q Fetch gap statistic for given stream and time window from local CAPS server echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool -G Fetch segment statistic for given stream and time window from local CAPS server using a gap threshold of 0.5 seconds echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool -G --tolerance=0.5 Fetch disk usage statistic for given stream and time window from local CAPS server echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool --print-stat Purge data for given stream and time window from local CAPS server echo "2022,05,01,12,00,00 2022,05,03,00,00,00 NET * * *" | capstool --purge """ import getopt import getpass import os import re import socket import ssl import struct import sys import traceback from datetime import datetime, timedelta from io import BytesIO from typing import BinaryIO, List, NamedTuple, TextIO, Tuple FullTimeFormat = "%F %T.%fZ" CAPSTimeRegex = re.compile( r"^\d{4},\d{1,2},\d{1,2}(,\d{1,2}(,\d{1,2}(,\d{1,2}(,\d{1,6})?)?)?)?$" ) PIPE = "-" ERR_OK = 0 ERR_UNKNOWN = 1 ERR_USAGE = 2 ERR_INPUT = 3 ERR_CONNECTION = 4 ERR_SERVER = 5 class CAPSToolError(Exception): def __init__(self, message: str, error_code: int): super().__init__(message) self.error_code: int = error_code class CAPSToolUsageError(CAPSToolError): def __init__(self, message: str): super().__init__(message, ERR_USAGE) class CAPSToolInputError(CAPSToolError): def __init__(self, message: str): super().__init__(message, ERR_INPUT) class CAPSToolConnectionError(CAPSToolError): def __init__(self, message: str): super().__init__(message, ERR_CONNECTION) class CAPSToolServerError(CAPSToolError): def __init__(self, message: str): super().__init__(message, ERR_SERVER) def py3bstr(s: str) -> bytes: """string to bytes""" return s.encode("utf-8") def py3ustr(b: bytes) -> str: """bytes to bytes""" return b.decode("utf-8", "replace") def error(msg: str) -> None: print(f"[error] {msg}", file=sys.stderr) def warning(msg: str) -> None: print(f"[warning] {msg}", file=sys.stderr) def info(msg: str) -> None: print(f"[info] {msg}", file=sys.stderr) def send(sock: socket.socket, data: str) -> None: sock.send(py3bstr(data) + b"\n") def read_buffer(sock: socket.socket, bufsize: int) -> bytearray: data = bytearray() while bufsize > 0: req: int = min(1024, bufsize) buf: bytes = sock.recv(req) if len(buf) == 0: break data += buf bufsize -= len(buf) return data def dataSize(fmt: str) -> Tuple[int, str]: if fmt == "RAW/INT8": return (1, "c") if fmt == "RAW/INT16": return (2, "h") if fmt == "RAW/INT32": return (4, "i") if fmt == "RAW/INT64": return (8, "q") if fmt == "RAW/FLOAT": return (4, "f") if fmt == "RAW/DOUBLE": return (8, "d") return 0, "" class Session(NamedTuple): sid: str = "" sfreq: str = "" uom: str = "" fmt: str = "" def __bool__(self) -> bool: return bool(self.sid) def writeRAW(out: TextIO, buf: bytearray, session: Session) -> int: """ Write RAW record in SLIST format to output file """ startTime = unpackTime(buf).strftime("%Y-%m-%dT%H:%M:%S.%f") size, t = dataSize(session.fmt) if not size: error(f"unsupported datatype: {session.fmt}") return 0 headerLen = 12 dataLen = len(buf) - headerLen sampleCount = dataLen // size if sampleCount <= 0: return 0 sid = session.sid.replace(".", "_") dataType = session.fmt.split("/")[1] toks = session.sfreq.split("/") freq = int(toks[0]) / int(toks[1]) if freq.is_integer(): freq = int(freq) # TIMESERIES CX_PB11__BHZ_R, 1588 samples, 40 sps, 2019-05-19T19:10:53.225000, # SLIST, FLOAT, M/S print( f"TIMESERIES {sid}_R, {sampleCount} samples, {freq} sps, {startTime}, SLIST, " f"{dataType}, {session.uom}", file=out, ) for i in range(sampleCount): ofs = i * size + headerLen value = struct.unpack(t, buf[ofs : ofs + size])[0] print(value, file=out) return sampleCount def writeHeli(out: TextIO, buf: bytearray, session: Session) -> int: """ Write HELI in SLIST format to output file Each HELI sample consists of 3 double values: start time (seconds since epoch), minimum, maximum The expected sample rate is 1s. We double the rate and write the minimum and aximum value as alternating values. """ recLen = 3 * 8 bufLen = len(buf) sampleCount = bufLen // recLen if sampleCount <= 0: return 0 secs, minimum, maximum = struct.unpack(" datetime: secs, usecs = struct.unpack(" None: self.session_by_id: dict[int, Session] = {} self.id_by_sid: dict[str, int] = {} def _handleRequests(self, lines: List[str]) -> None: d: dict[str, str] = {} for line in lines: if line == "END": break start = 0 pos = 0 while True: pos = line.find(":", start) if pos == -1: break key = line[start:pos] start = pos + 1 pos = line.find(",", start) if pos == -1: value = line[start:] else: value = line[start:pos] start = pos + 1 d[key] = value if "ID" in d: sessionID = int(d["ID"]) sid = d["SID"] if sessionID == -1: sessionID = self.id_by_sid[sid] del self.id_by_sid[sid] del self.session_by_id[sessionID] else: session = Session(sid, d["SFREQ"], d["UOM"], d["FMT"]) self.id_by_sid[sid] = sessionID self.session_by_id[sessionID] = session def handleResponse(self, sock: socket.socket) -> Tuple[bytearray, Session]: # read initial response from server buf = read_buffer(sock, 6) if len(buf) != 6: raise CAPSToolServerError("invalid response from server") (sessionID, size) = struct.unpack("=HI", buf) buf = read_buffer(sock, size) if sessionID == 0: resp = py3ustr(buf) if resp.startswith("ERROR:"): raise CAPSToolServerError(f"server responded with: {resp.strip()}") toks = resp.split("\n") # end of data if toks[0] == "EOD": return bytearray(), Session() # session table modified if toks[0] == "REQUESTS": self._handleRequests(toks[1:]) elif toks[0] != "STATUS OK": warning(f"server responded with unknown key word: {toks[0]}") return buf, Session() # unknown sessionID if sessionID not in self.session_by_id: raise CAPSToolServerError( f"server responded with unknown session id: {sessionID}" ) # read packet data return buf, self.session_by_id[sessionID] class CAPSTool: class SEEDParams: def __init__(self) -> None: self.enable = False self.resp_dict = False def __init__(self) -> None: self.host = "localhost" self.port = 18002 self.useSSL = False self.username = "" self.password = "" self.pingServer = False self.infoStreams = "" self.infoServer = False self.mtime = "" self.inputFiles = [PIPE] self.output = PIPE self.anyDateFormat = "%Y%m%d_%H%M%S" self.tmpOutputFile = "" self.printGaps = False self.printSegments = False self.printStat = False self.tolerance = 0 self.resolution = 0 self.real_time = False self.ooo = False self.heli = False self.itaper = "" self.bandpass = "" self.meta = False self.formatVersion = "" self.purge = False self.force = False self.SEED = self.SEEDParams() self._stdin_dirty = False def parse_args(self, args: List[str]) -> None: try: opts, args = getopt.gnu_getopt( args, "H:sc:PQI:XGSR:o:t:MDv:", [ "ping", "host=", "ssl", "credentials=", "ping", "info-streams=", "filter-list=", "mtime=", "info-server", "modified-after=", "print-gaps", "print-segments", "output-file=", "any-date-format=", "temp-file=", "rt", "tolerance=", "resolution=", "ooo", "out-of-order", "heli", "itaper=", "bandpass=", "meta", "version=", "purge", "force", "print-stat", ], ) except getopt.GetoptError as err: # will print something like "option -a not recognized" raise CAPSToolUsageError(str(err)) from err infoCommands = set() downloadOpts = set() for o, a in opts: if o in ["-H", "--host"]: toks = a.split(":", 1) if toks[0]: self.host = toks[0] if len(toks) == 2 and toks[1]: try: self.port = int(toks[1]) except ValueError as e: raise CAPSToolUsageError( f"invalid port given: {toks[1]}" ) from e elif o in ["-s", "--ssl"]: self.useSSL = True elif o in ["-c", "--credentials"]: toks = a.split(":", 1) if len(toks) > 1: self.username = toks[0] self.password = toks[1] else: self.username = toks[0] # ask for password self.password = getpass.getpass() elif o in ["-P", "--ping"]: self.pingServer = True infoCommands.add("PING") elif o in ["-Q"]: self.infoStreams = "*" infoCommands.add("INFO STREAMS") elif o in ["-I", "--info-streams", "--filter-list"]: self.infoStreams = a if a else "*" infoCommands.add("INFO STREAMS") elif o in ["--mtime"]: toks = a.split(":") if len(toks) != 2: raise CAPSToolUsageError( "invalid numbers of colons in --mtime parameter" ) if toks[0] and not CAPSTimeRegex.match(toks[0]): raise CAPSToolUsageError("invalid start time in --mtime parameter") if toks[1] and not CAPSTimeRegex.match(toks[1]): raise CAPSToolUsageError("invalid end time in --mtime parameter") self.mtime = a elif o in ["-X", "--info-server"]: infoCommands.add("INFO SERVER") elif o in ["--modified-after"]: if not CAPSTimeRegex.match(a): raise CAPSToolUsageError("invalid time in --modified-after") self.mtime = a elif o in ["-G", "--print-gaps"]: self.printGaps = True infoCommands.add("GAPS") elif o in ["-S", "--print-segments"]: self.printSegments = True infoCommands.add("SEGMENTS") elif o in ["--print-stat"]: self.printStat = True infoCommands.add("STATS") elif o in ["--tolerance"]: try: self.tolerance = int(round(float(a) * 10**6)) except ValueError as e: raise CAPSToolUsageError( "invalid floating point value in --tolerance parameter" ) from e elif o in ["-R", "--resolution"]: if not a.isdigit(): raise CAPSToolUsageError( "invalid integer value in --resolution parameter" ) self.resolution = int(a) elif o in ["-o", "--output-file"]: self.output = a downloadOpts.add(o) elif o in ["--any-date-format"]: self.anyDateFormat = a downloadOpts.add(o) elif o in ["-t", "--temp-file"]: self.tmpOutputFile = a downloadOpts.add(o) elif o in ["--rt"]: self.real_time = True downloadOpts.add(o) elif o in ["--ooo", "--out-of-order"]: self.ooo = True downloadOpts.add(o) elif o in ["-D", "--heli"]: self.heli = True downloadOpts.add(o) elif o in ["--itaper"]: self.itaper = a downloadOpts.add(o) elif o in ["--bandpass"]: self.bandpass = a downloadOpts.add(o) elif o in ["-M", "--meta"]: self.meta = True downloadOpts.add(o) elif o in ["-v", "--version"]: self.formatVersion = a downloadOpts.add(o) elif o in ["--purge"]: self.purge = True elif o in ["--force"]: self.force = True else: raise CAPSToolUsageError(f"unhandled option: {o}") # check for invalid command combinations if len(infoCommands) > 1: raise CAPSToolUsageError( f"more than one info command requested: {', '.join(infoCommands)}" ) # read request file arguments if len(args) > 0: self.inputFiles = args if self.pingServer: raise CAPSToolUsageError( "PING command and request file(s) may not be combined" ) if self.infoStreams: raise CAPSToolUsageError( "INFO STREAMS command and request file(s) may not be combined" ) # check for info command and download option combinations if infoCommands and downloadOpts: raise CAPSToolUsageError( "info commands and download options may not be combined\n" f" info command(s) : {', '.join(infoCommands)}\n" f" download option(s): {', '.join(downloadOpts)}" ) # check for invalid parameter combinations if self.tmpOutputFile and self.output is PIPE: raise CAPSToolUsageError( "temporary output file not supported when writing to stdout" ) if self.tolerance and not (self.printGaps or self.printSegments): raise CAPSToolUsageError( "tolerance is only supported in GAPS and SEGMENTS requests" ) if self.resolution and not (self.printGaps or self.printSegments): raise CAPSToolUsageError( "resolution is only supported in GAPS and SEGMENTS requests" ) if self.formatVersion and not self.meta: raise CAPSToolUsageError( "format version is currently only supported in META requests" ) if self.heli: if self.meta: raise CAPSToolUsageError("HELI and META requests may not be combined") elif self.itaper: raise CAPSToolUsageError( "itaper parameter only supported for HELI requests" ) elif self.bandpass: raise CAPSToolUsageError( "bandpass parameter only supported for HELI requests" ) if self.mtime and self.real_time: raise CAPSToolUsageError( "mtime data filter not available in real-time mode" ) def connect(self) -> socket.socket: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: if self.useSSL: context = ssl.SSLContext() sock = context.wrap_socket(sock) sock.connect((self.host, self.port)) except socket.error as e: raise CAPSToolConnectionError( f"could not establish connection to {self.host}:{self.port}" ) from e info(f"connected to {self.host}:{self.port}") return sock @staticmethod def disconnect(sock: socket.socket) -> None: send(sock, "BYE") sock.close() def authenticate(self, sock: socket.socket) -> None: if self.username: send(sock, f"AUTH {self.username} {self.password}") def beginRequest(self, sock: socket.socket) -> None: send(sock, "BEGIN REQUEST") if self.mtime: send(sock, f"MTIME {self.mtime}") if self.printGaps: send(sock, "GAPS ON") if self.printSegments: send(sock, "SEGMENTS ON") if self.printStat: send(sock, "STAT") if self.tolerance: send(sock, f"TOLERANCE {self.tolerance}") if self.resolution: send(sock, f"RESOLUTION {self.resolution}") if not self.real_time: send(sock, "REALTIME OFF") if self.ooo: send(sock, "OUTOFORDER ON") if self.meta: if self.formatVersion: send(sock, f"META@{self.formatVersion} ON") else: send(sock, "META ON") @staticmethod def endRequest(sock: socket.socket) -> None: send(sock, "END") @staticmethod def readDefaultResponse(sock: socket.socket) -> None: buf = read_buffer(sock, 6) if len(buf) != 6: raise CAPSToolServerError("unexpected response from server (len != 6)") (code, size) = struct.unpack("=HI", buf) if code != 0: raise CAPSToolServerError("unexpected response from server (id != 0)") if size < 0: raise CAPSToolServerError( "unexpected response from server (negative data size)" ) buf = read_buffer(sock, size) print(py3ustr(buf)) def addRequestLinesFromFile(self, buffer: BinaryIO, f: TextIO) -> None: def handleError(reason: str) -> None: msg = f"invalid request line #{lc}, {reason}" if f.isatty(): error(msg) else: print(f" > {line}", file=sys.stderr) raise CAPSToolInputError(msg) if self.heli: heli_filter = "" if self.itaper: heli_filter += f" ITAPER {self.itaper}" if self.bandpass: heli_filter += f" BANDPASS {self.bandpass}" lc = 0 for line in f: lc += 1 line = line.strip() if not line or line[0] == "#": print(f" + {lc:>5}: SKIP", file=sys.stderr) continue toks = line.split() if len(toks) < 5: handleError("less than 5 columns given") continue if toks[0] == "_": startTime = "" else: if CAPSTimeRegex.match(toks[0]): startTime = toks[0] else: handleError("invalid start time in column 1") continue if toks[1] == "_": endTime = "" else: if CAPSTimeRegex.match(toks[1]): endTime = toks[1] else: handleError("invalid end time in column 2") continue print(f" + {lc:>5}: OK", file=sys.stderr) # pylint: disable=consider-using-f-string if len(toks) == 5: streamID = "{}.{}..{}".format(*toks[2:5]) else: streamID = "{}.{}.{}.{}".format(*toks[2:6]) if self.heli: buffer.write(py3bstr(f"HELI ADD {streamID}{heli_filter}\n")) else: buffer.write(py3bstr(f"STREAM ADD {streamID}\n")) buffer.write(py3bstr(f"TIME {startTime}:{endTime}\n")) def prompt_for_confirmation(self, text: str) -> bool: if self._stdin_dirty: os.close(0) tty = "/dev/tty" if os.open(tty, os.O_RDONLY): raise OSError(f"could not open {tty}") self._stdin_dirty = False while True: response = input(text).lower() if response in ["yes", "y"]: return True if response in ["no", "n"]: return False print(" Respond with 'y' or 'n'.") def requestLines(self) -> BytesIO: buffer = BytesIO() # read request lines from stdin or files(s) if PIPE in self.inputFiles: if sys.stdin.isatty(): print( """Please input request lines of FORMAT followed by Ctrl+D FORMAT: starttime endtime NET STA [LOC] CHA Example: 2022,05,01,12,00,00 2022,05,03,00,00,00 AM R0F05 * *""", file=sys.stderr, ) else: info("reading request lines from stdin:") self.addRequestLinesFromFile(buffer, sys.stdin) self._stdin_dirty = True else: for file in self.inputFiles: info(f"reading request lines from {file}:") try: with open(file, "r", encoding="utf8") as f: self.addRequestLinesFromFile(buffer, f) except IOError as e: raise CAPSToolInputError( f"could not read request file {file}" ) from e return buffer def cmdPing(self) -> None: sock = self.connect() send(sock, "HELLO") self.readDefaultResponse(sock) self.disconnect(sock) def cmdInfoStreams(self) -> None: sock = self.connect() self.authenticate(sock) send(sock, f"INFO STREAMS {self.infoStreams}") self.readDefaultResponse(sock) self.disconnect(sock) def cmdInfoServer(self) -> None: sock = self.connect() self.authenticate(sock) mtime = f" MODIFIED AFTER {self.mtime}" if self.mtime else "" send(sock, f"INFO SERVER{mtime}") self.readDefaultResponse(sock) self.disconnect(sock) def cmdSegments(self) -> None: buffer = self.requestLines() sock = self.connect() self.authenticate(sock) self.beginRequest(sock) sock.send(buffer.getvalue()) self.endRequest(sock) # pylint: disable=consider-using-f-string print(f"{'Stream ID': <15} {'Start': <27} {'End': <27}") sessionTable = SessionTable() while True: buf, session = sessionTable.handleResponse(sock) if not buf: # EOD break if not session: continue start = unpackTime(buf) end = unpackTime(buf, 12) print( "{0: <15} {1: <27} {2: <27}".format( session.sid, start.strftime(FullTimeFormat), end.strftime(FullTimeFormat), ) ) def cmdStat(self) -> None: buffer = self.requestLines() sock = self.connect() self.authenticate(sock) self.beginRequest(sock) sock.send(buffer.getvalue()) self.endRequest(sock) self.readDefaultResponse(sock) self.disconnect(sock) def cmdPurge(self) -> None: buffer = self.requestLines() if not self.force and not self.prompt_for_confirmation( "Do you really want to remove the selected streams permanently from CAPS " "archive (y/n)? " ): return sock = self.connect() self.authenticate(sock) send(sock, "BEGIN PURGE") sock.send(buffer.getvalue()) self.endRequest(sock) self.readDefaultResponse(sock) self.disconnect(sock) def cmdDownload(self) -> None: # pylint: disable=consider-using-with # initialize out variables which depend on Python version used mseed_out = any_out = raw_out = meta_out = heli_out = None if not self.output or self.output == PIPE: write_to_stdout = True mseed_out = any_out = sys.stdout.buffer raw_out = meta_out = heli_out = sys.stdout mseed_file_name = any_file_name = raw_file_name = meta_file_name = ( heli_file_name ) = "stdout" else: write_to_stdout = False if self.output[-6:].lower() == ".mseed": mseed_file_name = self.output else: mseed_file_name = self.output + ".mseed" any_time = f"_{self.anyDateFormat}" if self.anyDateFormat else "" any_file_name = f"files of form {self.output}_STREAMID{any_time}.TYPE" if self.output[-4:].lower() == ".raw": raw_file_name = self.output else: raw_file_name = self.output + ".raw" if self.output[-5:].lower() == ".meta": meta_file_name = self.output else: meta_file_name = self.output + ".meta" if self.output[-5:].lower() == ".heli": heli_file_name = self.output else: heli_file_name = self.output + ".heli" mseed_bytes = 0 any_bytes = 0 raw_samples = 0 heli_samples = 0 meta_records = 0 buffer = self.requestLines() sock = self.connect() self.authenticate(sock) self.beginRequest(sock) sock.send(buffer.getvalue()) self.endRequest(sock) sessionTable = SessionTable() while True: buf, session = sessionTable.handleResponse(sock) if not buf: # EOD break if not session: continue # MSEED: Write binary MSeed data to stdout or single output file if session.fmt == "MSEED": if not mseed_out: mseed_out = open(mseed_file_name, "wb") mseed_out.write(buf) mseed_bytes += len(buf) # ANY: Write binary data to stdout or individual output files elif session.fmt == "ANY": if any_out: any_out.write(buf) any_bytes += len(buf) continue type_len = 4 for i in range(0, 4): if buf[i] == 0: type_len = i break packetType = py3ustr(buf[0:type_len]).lower() year, yday, hour, minute, second, usec = struct.unpack( "=hHBBBI", buf[5:16] ) filename = f"{self.output}_{session.sid}" if self.anyDateFormat: time = datetime(year, 1, 1, hour, minute, second, usec) time += timedelta(days=yday) filename += f"_{time.strftime(self.anyDateFormat)}" ext = f".{packetType if packetType else 'any'}" if filename[-len(ext) :].lower() != ext: filename += ext if self.tmpOutputFile: with open(self.tmpOutputFile, "wb") as f: f.write(buf[31:]) os.rename(self.tmpOutputFile, filename) else: with open(filename, "wb") as f: f.write(buf[31:]) any_bytes += len(buf) - 31 # RAW: Write SLIST string data to stdout or single output file elif session.fmt[:3] == "RAW": if not raw_out: raw_out = open(raw_file_name, "w", encoding="utf8") raw_samples += writeRAW(raw_out, buf, session) # HELI: Write SLIST string data to stdout or single output file elif session.fmt[:4] == "HELI": if not heli_out: heli_out = open(heli_file_name, "w", encoding="utf8") heli_samples = writeHeli(heli_out, buf, session) # META: Write string data to stdout or single output file elif session.fmt == "META" or session.fmt[:5] == "META@": version = 1 if len(session.fmt) > 4: try: version = int(session.fmt[5:]) except ValueError: error( f"Invalid META record version number '{session.fmt[5:]}' " f"for SID '{session.sid}'" ) continue if version > 2: error( f"Unsupported META record version number '{version}' for SID " f"'{session.sid}'" ) continue if version > 1: arrivalTime = unpackTime(buf, 24).strftime(FullTimeFormat) else: arrivalTime = "" if not meta_out: meta_out = open(meta_file_name, "w", encoding="utf8") # print header for first record if not meta_records: print( "#Stream ID|Sampling Frequency|Unit of Measurement" "|Start Time|End Time|Arrival Time", file=meta_out, ) print( f"{session.sid}|{session.sfreq}|{session.uom}|" f"{unpackTime(buf, 0).strftime(FullTimeFormat)}|" f"{unpackTime(buf, 12).strftime(FullTimeFormat)}|{arrivalTime}", file=meta_out, ) meta_records += 1 # unsupported session format else: error(f"Unsupported session FMT '{session.fmt}' in SID '{session.sid}'") # cleanup: flush buffers, close files if write_to_stdout: sys.stdout.flush() else: for fd in [mseed_out, raw_out, meta_out, heli_out]: if fd: fd.close() if any((mseed_bytes, any_bytes, raw_samples, heli_samples, meta_records)): msg = "Data acquisition completed:" if mseed_bytes: msg += f"\n + MSEED: Wrote {mseed_bytes} bytes to {mseed_file_name}" if any_bytes: msg += f"\n + ANY : Wrote {any_bytes} bytes to {any_file_name}" if raw_samples: msg += f"\n + RAW : Wrote {raw_samples} samples to {raw_file_name}" if heli_samples: msg += f"\n + HELI : Wrote {heli_samples} samples to {heli_file_name}" if meta_records: msg += f"\n + META : Wrote {meta_records} records to {meta_file_name}" info(msg) else: info("No data written. Check request.") self.disconnect(sock) def __call__(self) -> None: if self.pingServer: self.cmdPing() elif self.infoStreams: self.cmdInfoStreams() elif self.infoServer: self.cmdInfoServer() elif self.printSegments or self.printGaps: self.cmdSegments() elif self.printStat: self.cmdStat() elif self.purge: self.cmdPurge() else: self.cmdDownload() def main(args: List[str]) -> None: if "-h" in args or "--help" in args: print(__doc__, file=sys.stderr) return app = CAPSTool() app.parse_args(args) app() if __name__ == "__main__": exit_code = ERR_OK try: main(sys.argv[1:]) except KeyboardInterrupt: error("interrupted") except EOFError: error("end of file") except CAPSToolError as main_e: error(str(main_e)) exit_code = main_e.error_code except Exception as main_e: error(str(main_e)) traceback.print_exc(file=sys.stderr) exit_code = ERR_UNKNOWN sys.exit(exit_code)