[installation] Init with inital config for global
This commit is contained in:
421
bin/data2caps
Executable file
421
bin/data2caps
Executable file
@ -0,0 +1,421 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import getopt
|
||||
import os
|
||||
import sys
|
||||
import signal
|
||||
|
||||
from fractions import Fraction
|
||||
|
||||
import numpy as np
|
||||
|
||||
from gempa import CAPS
|
||||
|
||||
|
||||
class Record:
|
||||
net = ""
|
||||
sta = ""
|
||||
loc = ""
|
||||
cha = ""
|
||||
samples = []
|
||||
start = CAPS.Time.GMT()
|
||||
numerator = 0
|
||||
denominator = 1
|
||||
nSamp = 0
|
||||
unit = ""
|
||||
|
||||
|
||||
usage_info = f"""Usage:
|
||||
|
||||
{os.path.basename(__file__)} [options]
|
||||
|
||||
Push data into CAPS. The input and output formats can be easily adjusted.
|
||||
|
||||
Options:
|
||||
-H, --host Host name of CAPS server to connect to. Default: localhost:18003.
|
||||
-h, --help Display this help message and exit.
|
||||
-i, --input Name of input file. Create a generic signal if not given.
|
||||
-f, --format Format of input data. Supported: slist, unavco.
|
||||
-m, --multiplier Multiplier applied to data samples for generating integers.
|
||||
-n, --network Network code of data to be sent. Read from input if not given.
|
||||
|
||||
Examples:
|
||||
|
||||
Generate generic signal assuming AM network code and send to a caps server with default port 18003
|
||||
{os.path.basename(__file__)} -n AM -H localhost:18003
|
||||
|
||||
Read data file in slist format, upscale values by 1000000000 and send in RAW format to the default caps server
|
||||
{os.path.basename(__file__)} -i file.slist -f slist -m 1000000000
|
||||
|
||||
Read data file in unavco 1.0 format and send in RAW format. The network code must be specified.
|
||||
{os.path.basename(__file__)} -f unavco -i unavco-pressure-v1-0.txt -n AB
|
||||
|
||||
Read data file in unavco 1.1 format and send in RAW format
|
||||
{os.path.basename(__file__)} -f unavco -i unavco-tilt-v1-1.txt
|
||||
"""
|
||||
|
||||
|
||||
def usage(exitcode=0):
|
||||
sys.stderr.write(usage_info)
|
||||
return exitcode
|
||||
|
||||
|
||||
output = CAPS.Plugin("data2caps")
|
||||
|
||||
|
||||
def readSlist(filePath, multiplier=1.0, network=None):
|
||||
# supported formats:
|
||||
# TIMESERIES AM_R1108_00_SHZ_R, 8226 samples, 50 sps, 2018-04-10T10:20:03.862000, SLIST, FLOAT, M/S**2
|
||||
# 0.000134157
|
||||
# ...
|
||||
data = Record()
|
||||
samples = []
|
||||
all_data = []
|
||||
try:
|
||||
with open(filePath, "r", encoding="utf8") as file:
|
||||
all_data = [line.strip() for line in file.readlines()]
|
||||
except FileNotFoundError:
|
||||
print(f"File '{filePath}' not found.", file=sys.stderr)
|
||||
return False
|
||||
|
||||
if len(all_data) == 0:
|
||||
print(f"No data read from file {filePath}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
header = all_data[0]
|
||||
dataType = header.split(",")[0].split(" ")[0]
|
||||
if dataType != "TIMESERIES":
|
||||
print(
|
||||
f"Unsupported data type {dataType} in {filePath}. Supported: TIMESERIES",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
|
||||
if network:
|
||||
data.net = network
|
||||
else:
|
||||
data.net = header.split(",")[0].split(" ")[1].split("_")[0]
|
||||
data.sta = header.split(",")[0].split(" ")[1].split("_")[1]
|
||||
data.loc = header.split(",")[0].split(" ")[1].split("_")[2]
|
||||
data.cha = header.split(",")[0].split(" ")[1].split("_")[3]
|
||||
data.nSamp = int(header.split(",")[1].split(" ")[-2])
|
||||
|
||||
try:
|
||||
frac = Fraction(header.split(",")[2].split(" ")[-2]).limit_denominator()
|
||||
except Exception:
|
||||
print("Unknown sample rate", file=sys.stderr)
|
||||
return False
|
||||
|
||||
data.numerator = frac.numerator
|
||||
data.denominator = frac.denominator
|
||||
|
||||
time = header.split(",")[3].strip(" ")
|
||||
data.start = CAPS.Time.FromString(time, "%FT%T.%f")
|
||||
data.unit = header.split(",")[6].strip(" ")
|
||||
samples = [float(x) for x in all_data[1:]]
|
||||
min_abs = min(abs(x) for x in samples)
|
||||
max_abs = max(abs(x) for x in samples)
|
||||
print(
|
||||
f" + minimum/maximum absolute value found in data: {min_abs}/{max_abs}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(f" + applying multuplier: {multiplier}", file=sys.stderr)
|
||||
for sample in samples:
|
||||
if dataType == "TIMESERIES":
|
||||
data.samples.append(int(round(sample * multiplier, 0)))
|
||||
|
||||
min_abs = min(abs(x) for x in data.samples)
|
||||
max_abs = max(abs(x) for x in data.samples)
|
||||
print(
|
||||
" + minimum/maximum absolute value after applying multiplier: "
|
||||
f"{min_abs}/{max_abs}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def readUnavco(filePath, multiplier=1.0, network=None):
|
||||
# supported formats:
|
||||
# version 1.1
|
||||
# # PBO Tiltmeter Data, format version 1.1
|
||||
# # http://pboweb.unavco.org/strain_data
|
||||
# #
|
||||
# # B201 coldwt201bwa2007
|
||||
# # Latitude : +46.3033 degrees (WGS-84)
|
||||
# # Longitude : -122.2648 degrees (WGS-84)
|
||||
# # Elevation : 990 meters (WGS-84)
|
||||
# # Sensor Depth : 24.38 meters
|
||||
# # X Orientation: 316 degrees East of North (magnetic)
|
||||
# # SEED Codes : PB B201 TT LAE
|
||||
# #
|
||||
# # Timestamp (UTC) Tilt X (microradians)
|
||||
# 2023-10-05T16:00:00.58 162.981
|
||||
|
||||
data = Record()
|
||||
version = None
|
||||
versionSupported = ["1.0", "1.1"]
|
||||
all_data = []
|
||||
|
||||
try:
|
||||
with open(filePath, "r", encoding="utf8") as file:
|
||||
# read header
|
||||
all_data = [line.strip().split(" ") for line in file.readlines()]
|
||||
except FileNotFoundError:
|
||||
print(f"File '{filePath}' not found.", file=sys.stderr)
|
||||
return False
|
||||
|
||||
if len(all_data) == 0:
|
||||
print(f"No data read from file {filePath}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
cnt = 0
|
||||
for line in all_data:
|
||||
if not line[0].startswith("#"):
|
||||
break
|
||||
|
||||
try:
|
||||
if "version" in line[-2]:
|
||||
version = line[-1]
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
cnt += 1
|
||||
|
||||
if version not in versionSupported:
|
||||
print(
|
||||
f"Unsupported format version '{version}' of file '{filePath}'. "
|
||||
f"Supported are: {versionSupported}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
|
||||
print(f" + unavco format version: {version}", file=sys.stderr)
|
||||
|
||||
# extract meta data
|
||||
header = all_data[:cnt]
|
||||
for line in header:
|
||||
# stream code
|
||||
if set(["seed", "codes"]).issubset(set(x.lower() for x in line)):
|
||||
if network:
|
||||
data.net = network
|
||||
else:
|
||||
if version == "1.0":
|
||||
if not network:
|
||||
print(
|
||||
f"Found version {version}: Must provide network code",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
elif version == "1.1":
|
||||
data.net = line[-4]
|
||||
else:
|
||||
print(
|
||||
f"Unsupported format version {version}. Supported are: {version}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
data.sta = line[-3]
|
||||
data.loc = line[-2]
|
||||
data.cha = line[-1]
|
||||
|
||||
# physical unit
|
||||
if "Timestamp" in line[1]:
|
||||
unit = line[-1].strip("(").strip(")")
|
||||
if "microradians" in unit:
|
||||
if multiplier == 1000.0:
|
||||
unit = "nRad"
|
||||
print(f" + converting data in microradians to {unit}", file=sys.stderr)
|
||||
|
||||
elif multiplier == 1.0:
|
||||
unit = "nRad"
|
||||
print(
|
||||
f" + converting data from microradians to {unit} for high precision",
|
||||
file=sys.stderr,
|
||||
)
|
||||
multiplier = 1000
|
||||
|
||||
if "hPa" in unit:
|
||||
if multiplier == 1.0:
|
||||
multiplier = 100
|
||||
unit = "Pa"
|
||||
print(f" + converting data from hPa to {unit}", file=sys.stderr)
|
||||
|
||||
data.unit = unit
|
||||
|
||||
if version == "1.0":
|
||||
time1 = CAPS.Time.FromString(all_data[cnt][0] + all_data[cnt][1], "%F%T.%f")
|
||||
time2 = CAPS.Time.FromString(
|
||||
all_data[cnt + 1][0] + all_data[cnt + 1][1], "%F%T.%f"
|
||||
)
|
||||
elif version == "1.1":
|
||||
time1 = CAPS.Time.FromString(all_data[cnt][0], "%FT%T.%f")
|
||||
time2 = CAPS.Time.FromString(all_data[cnt + 1][0], "%FT%T.%f")
|
||||
else:
|
||||
return False
|
||||
|
||||
data.start = time1
|
||||
try:
|
||||
frac = Fraction(1.0 / (time2 - time1).length()).limit_denominator()
|
||||
except Exception:
|
||||
print("Unknown sample rate", file=sys.stderr)
|
||||
return False
|
||||
|
||||
data.numerator = frac.numerator
|
||||
data.denominator = frac.denominator
|
||||
# extract time series
|
||||
for i in all_data[cnt:]:
|
||||
if version == "1.0":
|
||||
data.samples.append(int(round(float(i[2]) * multiplier, 0)))
|
||||
elif version == "1.1":
|
||||
data.samples.append(int(round(float(i[1]) * multiplier, 0)))
|
||||
else:
|
||||
return False
|
||||
|
||||
data.nSamp = len(data.samples)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def signal_handler(sig, frame): # pylint: disable=W0613
|
||||
print("Caught Ctrl+C!", file=sys.stderr)
|
||||
output.quit()
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def main():
|
||||
inputFormat = None
|
||||
inputFile = None
|
||||
host = "localhost"
|
||||
port = 18003
|
||||
multiplier = 1
|
||||
network = None
|
||||
|
||||
try:
|
||||
opts, _ = getopt.getopt(
|
||||
sys.argv[1:],
|
||||
"hH:i:f:m:n:",
|
||||
["help", "host=", "input=", "format=", "multiplier=", "network="],
|
||||
)
|
||||
except getopt.GetoptError as err:
|
||||
# print help information and exit:
|
||||
print(str(err)) # will print something like "option -a not recognized"
|
||||
return usage(2)
|
||||
|
||||
addr = None
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
for o, a in opts:
|
||||
if o in ["-h", "--help"]:
|
||||
return usage()
|
||||
|
||||
if o in ["-H", "--host"]:
|
||||
addr = a
|
||||
elif o in ["-i", "--input"]:
|
||||
inputFile = a
|
||||
elif o in ["-f", "--format"]:
|
||||
inputFormat = a
|
||||
elif o in ["-m", "--multiplier"]:
|
||||
multiplier = float(a)
|
||||
elif o in ["-n", "--network"]:
|
||||
network = a
|
||||
else:
|
||||
assert False, "unhandled option"
|
||||
|
||||
if addr:
|
||||
if ":" in addr:
|
||||
try:
|
||||
host, port = addr.split(":")
|
||||
except BaseException:
|
||||
print(f"Invalid host address given: {addr}\n", file=sys.stderr)
|
||||
return 1
|
||||
else:
|
||||
host = addr
|
||||
|
||||
if port:
|
||||
try:
|
||||
port = int(port)
|
||||
except BaseException:
|
||||
print(f"Invalid port given: {port}", file=sys.stderr)
|
||||
return 1
|
||||
else:
|
||||
port = 18003
|
||||
|
||||
if not host:
|
||||
host = "localhost"
|
||||
|
||||
output.setHost(host)
|
||||
output.setPort(port)
|
||||
output.setBufferSize(1 << 30)
|
||||
output.enableLogging()
|
||||
|
||||
res = None
|
||||
print("Summary", file=sys.stderr)
|
||||
print(f" + input file: {inputFile}", file=sys.stderr)
|
||||
print(f" + input format: {inputFormat}", file=sys.stderr)
|
||||
print(f" + caps server: {host}:{port}", file=sys.stderr)
|
||||
if network:
|
||||
print(f" + set network: {network}", file=sys.stderr)
|
||||
|
||||
# generic signal
|
||||
if not inputFormat and not inputFile:
|
||||
startTime = CAPS.Time.GMT()
|
||||
print("Sending generic test data", file=sys.stderr)
|
||||
x = np.array([1, 2], dtype=np.int32)
|
||||
|
||||
if not network:
|
||||
network = "AB"
|
||||
print(f"Assuming network '{network}'", file=sys.stderr)
|
||||
|
||||
res = output.push(
|
||||
network, "HMA", "", "BHZ", startTime, 1, 1, "m", x, CAPS.DT_INT32
|
||||
)
|
||||
|
||||
if res != CAPS.Plugin.Success:
|
||||
print("Failed to send packet", file=sys.stderr)
|
||||
|
||||
output.close()
|
||||
return 0
|
||||
|
||||
if not inputFile:
|
||||
print("Must provide input file", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# data from input file
|
||||
data = None
|
||||
if inputFormat == "slist":
|
||||
data = readSlist(inputFile, multiplier, network)
|
||||
elif inputFormat == "unavco":
|
||||
data = readUnavco(inputFile, multiplier, network)
|
||||
else:
|
||||
return 1
|
||||
|
||||
if not data:
|
||||
print(f"Error parsing {inputFile}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"Sending data to CAPS server {host}:{port}", file=sys.stderr)
|
||||
|
||||
res = output.push(
|
||||
data.net,
|
||||
data.sta,
|
||||
data.loc,
|
||||
data.cha,
|
||||
data.start,
|
||||
data.numerator,
|
||||
data.denominator,
|
||||
data.unit,
|
||||
data.samples,
|
||||
CAPS.DT_INT32,
|
||||
)
|
||||
|
||||
if res != CAPS.Plugin.Success:
|
||||
output.close()
|
||||
print("Failed to send packet", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user