610 lines
21 KiB
Python
610 lines
21 KiB
Python
################################################################################
|
|
# Copyright (C) 2013-2014 gempa GmbH
|
|
#
|
|
# RequestOptions -- HTTP GET request parameters
|
|
#
|
|
# Author: Stephan Herrnkind
|
|
# Email: herrnkind@gempa.de
|
|
################################################################################
|
|
|
|
import fnmatch
|
|
import math
|
|
import re
|
|
|
|
from twisted.web import http
|
|
|
|
from seiscomp.core import Time
|
|
import seiscomp.logging
|
|
import seiscomp.math
|
|
|
|
from .utils import u_str
|
|
|
|
|
|
class RequestOptions:
|
|
# the match() method matched only patterns at the beginning of a string,
|
|
# since we have to ensure that no invalid character is present we use the
|
|
# search() method in combination with a negated pattern instead
|
|
FloatChars = re.compile(r"[^-0-9.]").search
|
|
ChannelChars = re.compile(r"[^A-Za-z0-9*?]").search
|
|
ChannelExtChars = re.compile(r"[^A-Za-z0-9*?+\-_]").search
|
|
BooleanTrueValues = ["1", "true", "t", "yes", "y"]
|
|
BooleanFalseValues = ["0", "false", "f", "no", "n"]
|
|
OutputFormats = [] # override in derived classes
|
|
|
|
PStart = ["starttime", "start"]
|
|
PEnd = ["endtime", "end"]
|
|
PStartBefore = ["startbefore"]
|
|
PStartAfter = ["startafter"]
|
|
PEndBefore = ["endbefore"]
|
|
PEndAfter = ["endafter"]
|
|
SimpleTimeParams = PStart + PEnd
|
|
WindowTimeParams = PStartBefore + PStartAfter + PEndBefore + PEndAfter
|
|
TimeParams = SimpleTimeParams + WindowTimeParams
|
|
|
|
PNet = ["network", "net"]
|
|
PSta = ["station", "sta"]
|
|
PLoc = ["location", "loc"]
|
|
PCha = ["channel", "cha"]
|
|
StreamParams = PNet + PSta + PLoc + PCha
|
|
|
|
PMinLat = ["minlatitude", "minlat"]
|
|
PMaxLat = ["maxlatitude", "maxlat"]
|
|
PMinLon = ["minlongitude", "minlon"]
|
|
PMaxLon = ["maxlongitude", "maxlon"]
|
|
PLat = ["latitude", "lat"]
|
|
PLon = ["longitude", "lon"]
|
|
PMinRadius = ["minradius"]
|
|
PMaxRadius = ["maxradius"]
|
|
GeoRectParams = PMinLat + PMaxLat + PMinLon + PMaxLon
|
|
GeoCircleParams = PLat + PLon + PMinRadius + PMaxRadius
|
|
GeoParams = GeoRectParams + GeoCircleParams
|
|
|
|
PFormat = ["format"]
|
|
PNoData = ["nodata"]
|
|
OutputParams = PFormat + PNoData
|
|
|
|
POSTParams = OutputParams
|
|
GETParams = StreamParams + SimpleTimeParams
|
|
|
|
# ---------------------------------------------------------------------------
|
|
class Channel:
|
|
def __init__(self):
|
|
self.net = None
|
|
self.sta = None
|
|
self.loc = None
|
|
self.cha = None
|
|
|
|
def matchNet(self, value):
|
|
return self.match(value, self.net)
|
|
|
|
def matchSta(self, value):
|
|
return self.match(value, self.sta)
|
|
|
|
def matchLoc(self, value):
|
|
return self.match(value, self.loc, True)
|
|
|
|
def matchCha(self, value):
|
|
return self.match(value, self.cha)
|
|
|
|
@staticmethod
|
|
def match(value, globList, testEmpty=False):
|
|
if not globList:
|
|
return True
|
|
|
|
for glob in globList:
|
|
if testEmpty and value == "" and glob == "--":
|
|
return True
|
|
if fnmatch.fnmatchcase(value, glob):
|
|
return True
|
|
|
|
return False
|
|
|
|
# ---------------------------------------------------------------------------
|
|
class Time:
|
|
def __init__(self):
|
|
self.simpleTime = True
|
|
self.start = None
|
|
self.end = None
|
|
# window time only
|
|
self.startBefore = None
|
|
self.startAfter = None
|
|
self.endBefore = None
|
|
self.endAfter = None
|
|
|
|
# used by FDSN Station and DataSelect
|
|
def match(self, start, end=None):
|
|
# simple time: limit to epochs intersecting with the specified time
|
|
# range
|
|
res = (self.start is None or end is None or end >= self.start) and (
|
|
self.end is None or start <= self.end
|
|
)
|
|
|
|
# window time: limit to epochs strictly starting or ending before or
|
|
# after a specified time value
|
|
if not self.simpleTime:
|
|
res = (
|
|
res
|
|
and (
|
|
self.startBefore is None
|
|
or (start is not None and start < self.startBefore)
|
|
)
|
|
and (
|
|
self.startAfter is None
|
|
or (start is not None and start > self.startAfter)
|
|
)
|
|
and (
|
|
self.endBefore is None
|
|
or (end is not None and end < self.endBefore)
|
|
)
|
|
and (self.endAfter is None or end is None or end > self.endAfter)
|
|
)
|
|
|
|
return res
|
|
|
|
# ---------------------------------------------------------------------------
|
|
class Geo:
|
|
# -----------------------------------------------------------------------
|
|
class BBox:
|
|
def __init__(self):
|
|
self.minLat = None
|
|
self.maxLat = None
|
|
self.minLon = None
|
|
self.maxLon = None
|
|
|
|
def dateLineCrossing(self):
|
|
return self.minLon and self.maxLon and self.minLon > self.maxLon
|
|
|
|
# -----------------------------------------------------------------------
|
|
class BCircle:
|
|
def __init__(self):
|
|
self.lat = None
|
|
self.lon = None
|
|
self.minRad = None
|
|
self.maxRad = None
|
|
|
|
# -------------------------------------------------------------------
|
|
# Calculates outer bounding box
|
|
def calculateBBox(self):
|
|
def rad(degree):
|
|
return math.radians(degree)
|
|
|
|
def deg(radians):
|
|
return math.degrees(radians)
|
|
|
|
b = RequestOptions.Geo.BBox()
|
|
if self.maxRad is None or self.maxRad >= 180:
|
|
return b
|
|
|
|
b.minLat = self.lat - self.maxRad
|
|
b.maxLat = self.lat + self.maxRad
|
|
if b.minLat > -90 and b.maxLat < 90:
|
|
dLon = deg(
|
|
math.asin(math.sin(rad(self.maxRad) / math.cos(rad(self.lat))))
|
|
)
|
|
b.minLon = self.lon - dLon
|
|
if b.minLon < -180:
|
|
b.minLon += 360
|
|
b.maxLon = self.lon + dLon
|
|
if b.maxLon > 180:
|
|
b.maxLon -= 360
|
|
else:
|
|
# pole within distance: one latitude and no longitude
|
|
# restrictions remains
|
|
if b.minLat <= -90:
|
|
b.minLat = None
|
|
else:
|
|
b.maxLat = None
|
|
b.minLon = None
|
|
b.maxLon = None
|
|
|
|
return b
|
|
|
|
# -----------------------------------------------------------------------
|
|
def __init__(self):
|
|
self.bBox = None
|
|
self.bCircle = None
|
|
|
|
# -----------------------------------------------------------------------
|
|
def match(self, lat, lon):
|
|
if self.bBox is not None:
|
|
b = self.bBox
|
|
if b.minLat is not None and lat < b.minLat:
|
|
return False
|
|
if b.maxLat is not None and lat > b.maxLat:
|
|
return False
|
|
# date line crossing if minLon > maxLon
|
|
if b.dateLineCrossing():
|
|
return lon >= b.minLon or lon <= b.maxLon
|
|
if b.minLon is not None and lon < b.minLon:
|
|
return False
|
|
if b.maxLon is not None and lon > b.maxLon:
|
|
return False
|
|
return True
|
|
|
|
if self.bCircle:
|
|
c = self.bCircle
|
|
dist = seiscomp.math.delazi(c.lat, c.lon, lat, lon)
|
|
if c.minRad is not None and dist[0] < c.minRad:
|
|
return False
|
|
if c.maxRad is not None and dist[0] > c.maxRad:
|
|
return False
|
|
return True
|
|
|
|
return False
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def __init__(self):
|
|
self.service = ""
|
|
self.accessTime = Time.GMT()
|
|
self.userName = None
|
|
|
|
self.time = None
|
|
self.channel = None
|
|
self.geo = None
|
|
|
|
self.noData = http.NO_CONTENT
|
|
self.format = None
|
|
|
|
self._args = {}
|
|
self.streams = [] # 1 entry for GET, multipl
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseOutput(self):
|
|
# nodata
|
|
code = self.parseInt(self.PNoData)
|
|
if code is not None:
|
|
if code not in (http.NO_CONTENT, http.NOT_FOUND):
|
|
self.raiseValueError(self.PNoData[0])
|
|
self.noData = code
|
|
|
|
# format
|
|
key, value = self.getFirstValue(self.PFormat)
|
|
if value is None:
|
|
# no format specified: default to first in list if available
|
|
if len(self.OutputFormats) > 0:
|
|
self.format = self.OutputFormats[0]
|
|
else:
|
|
value = value.lower()
|
|
if value in self.OutputFormats:
|
|
self.format = value
|
|
else:
|
|
self.raiseValueError(key)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseChannel(self):
|
|
c = RequestOptions.Channel()
|
|
|
|
c.net = self.parseChannelChars(self.PNet, False, True)
|
|
c.sta = self.parseChannelChars(self.PSta)
|
|
c.loc = self.parseChannelChars(self.PLoc, True)
|
|
c.cha = self.parseChannelChars(self.PCha)
|
|
|
|
if c.net or c.sta or c.loc or c.cha:
|
|
self.channel = c
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseTime(self, parseWindowTime=False):
|
|
t = RequestOptions.Time()
|
|
|
|
# start[time], end[time]
|
|
t.start = self.parseTimeStr(self.PStart)
|
|
t.end = self.parseTimeStr(self.PEnd)
|
|
|
|
simpleTime = t.start is not None or t.end is not None
|
|
|
|
# [start,end][before,after]
|
|
if parseWindowTime:
|
|
t.startBefore = self.parseTimeStr(self.PStartBefore)
|
|
t.startAfter = self.parseTimeStr(self.PStartAfter)
|
|
t.endBefore = self.parseTimeStr(self.PEndBefore)
|
|
t.endAfter = self.parseTimeStr(self.PEndAfter)
|
|
|
|
windowTime = (
|
|
t.startBefore is not None
|
|
or t.startAfter is not None
|
|
or t.endBefore is not None
|
|
or t.endAfter is not None
|
|
)
|
|
if simpleTime or windowTime:
|
|
self.time = t
|
|
self.time.simpleTime = not windowTime
|
|
|
|
elif simpleTime:
|
|
self.time = t
|
|
self.time.simpleTime = True
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseGeo(self):
|
|
# bounding box (optional)
|
|
b = RequestOptions.Geo.BBox()
|
|
b.minLat = self.parseFloat(self.PMinLat, -90, 90)
|
|
b.maxLat = self.parseFloat(self.PMaxLat, -90, 90)
|
|
if b.minLat is not None and b.maxLat is not None and b.minLat > b.maxLat:
|
|
raise ValueError(f"{self.PMinLat[0]} exceeds {self.PMaxLat[0]}")
|
|
|
|
b.minLon = self.parseFloat(self.PMinLon, -180, 180)
|
|
b.maxLon = self.parseFloat(self.PMaxLon, -180, 180)
|
|
# maxLon < minLon -> date line crossing
|
|
|
|
hasBBoxParam = (
|
|
b.minLat is not None
|
|
or b.maxLat is not None
|
|
or b.minLon is not None
|
|
or b.maxLon is not None
|
|
)
|
|
|
|
# bounding circle (optional)
|
|
c = RequestOptions.Geo.BCircle()
|
|
c.lat = self.parseFloat(self.PLat, -90, 90)
|
|
c.lon = self.parseFloat(self.PLon, -180, 180)
|
|
c.minRad = self.parseFloat(self.PMinRadius, 0, 180)
|
|
c.maxRad = self.parseFloat(self.PMaxRadius, 0, 180)
|
|
if c.minRad is not None and c.maxRad is not None and c.minRad > c.maxRad:
|
|
raise ValueError(f"{self.PMinRadius[0]} exceeds {self.PMaxRadius[0]}")
|
|
|
|
hasBCircleRadParam = c.minRad is not None or c.maxRad is not None
|
|
hasBCircleParam = c.lat is not None or c.lon is not None or hasBCircleRadParam
|
|
|
|
# bounding box and bounding circle may not be combined
|
|
if hasBBoxParam and hasBCircleParam:
|
|
raise ValueError(
|
|
"bounding box and bounding circle parameters may not be combined"
|
|
)
|
|
if hasBBoxParam:
|
|
self.geo = RequestOptions.Geo()
|
|
self.geo.bBox = b
|
|
elif hasBCircleRadParam:
|
|
self.geo = RequestOptions.Geo()
|
|
if c.lat is None:
|
|
c.lat = 0.0
|
|
if c.lon is None:
|
|
c.lon = 0.0
|
|
self.geo.bCircle = c
|
|
|
|
# ---------------------------------------------------------------------------
|
|
@staticmethod
|
|
def _assertValueRange(key, v, minValue, maxValue):
|
|
if (minValue is not None and v < minValue) or (
|
|
maxValue is not None and v > maxValue
|
|
):
|
|
minStr, maxStr = "-inf", "inf"
|
|
if minValue is not None:
|
|
minStr = str(minValue)
|
|
if maxValue is not None:
|
|
maxStr = str(maxValue)
|
|
raise ValueError(f"parameter not in domain [{minStr},{maxStr}]: {key}")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
@staticmethod
|
|
def raiseValueError(key):
|
|
raise ValueError(f"invalid value in parameter: {key}")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def getFirstValue(self, keys):
|
|
for key in keys:
|
|
if key in self._args:
|
|
return key, self._args[key][0].strip()
|
|
|
|
return None, None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def getValues(self, keys):
|
|
v = []
|
|
for key in keys:
|
|
if key in self._args:
|
|
v += self._args[key]
|
|
return v
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def getListValues(self, keys, lower=False):
|
|
values = set()
|
|
for key in keys:
|
|
if key not in self._args:
|
|
continue
|
|
|
|
for vList in self._args[key]:
|
|
for v in vList.split(","):
|
|
if v is None:
|
|
continue
|
|
v = v.strip()
|
|
if lower:
|
|
v = v.lower()
|
|
values.add(v)
|
|
|
|
return values
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseInt(self, keys, minValue=None, maxValue=None):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
try:
|
|
i = int(value)
|
|
except ValueError as e:
|
|
raise ValueError(f"invalid integer value in parameter: {key}") from e
|
|
|
|
self._assertValueRange(key, i, minValue, maxValue)
|
|
return i
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseFloat(self, keys, minValue=None, maxValue=None):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
if self.FloatChars(value):
|
|
raise ValueError(
|
|
f"invalid characters in float parameter: {key} (scientific notation "
|
|
"forbidden by spec)"
|
|
)
|
|
|
|
try:
|
|
f = float(value)
|
|
except ValueError as e:
|
|
raise ValueError(f"invalid float value in parameter: {key}") from e
|
|
|
|
self._assertValueRange(key, f, minValue, maxValue)
|
|
return f
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseBool(self, keys):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
value = value.lower()
|
|
if value in self.BooleanTrueValues:
|
|
return True
|
|
if value in self.BooleanFalseValues:
|
|
return False
|
|
|
|
raise ValueError(f"invalid boolean value in parameter: {key}")
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseTimeStr(self, keys):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
time = Time.FromString(value)
|
|
# use explicit test for None here since bool value for epoch date
|
|
# (1970-01-01) is False
|
|
if time is None:
|
|
raise ValueError(f"invalid date format in parameter: {key}")
|
|
|
|
return time
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseChannelChars(self, keys, allowEmpty=False, useExtChars=False):
|
|
# channel parameters may be specified as a comma separated list and may
|
|
# be repeated several times
|
|
values = None
|
|
for vList in self.getValues(keys):
|
|
if values is None:
|
|
values = []
|
|
for v in vList.split(","):
|
|
v = v.strip()
|
|
if allowEmpty and (v == "--" or len(v) == 0):
|
|
values.append("--")
|
|
continue
|
|
|
|
if (useExtChars and self.ChannelExtChars(v)) or (
|
|
not useExtChars and self.ChannelChars(v)
|
|
):
|
|
raise ValueError(f"invalid characters in parameter: {keys[0]}")
|
|
values.append(v)
|
|
|
|
return values
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parseGET(self, args):
|
|
# transform keys to lower case
|
|
if args is not None:
|
|
for k, v in args.items():
|
|
k = u_str(k.lower())
|
|
if k not in self.GETParams:
|
|
raise ValueError(f"invalid param: {k}")
|
|
|
|
self._args[k] = [u_str(x) for x in v]
|
|
|
|
# ---------------------------------------------------------------------------
|
|
def parsePOST(self, content):
|
|
nLine = 0
|
|
|
|
for line in content:
|
|
nLine += 1
|
|
line = u_str(line.strip())
|
|
|
|
# ignore empty and comment lines
|
|
if len(line) == 0 or line[0] == "#":
|
|
continue
|
|
|
|
# collect parameter (non stream lines)
|
|
toks = line.split("=", 1)
|
|
if len(toks) > 1:
|
|
key = toks[0].strip().lower()
|
|
|
|
isPOSTParam = False
|
|
for p in self.POSTParams:
|
|
if p == key:
|
|
if key not in self._args:
|
|
self._args[key] = []
|
|
self._args[key].append(toks[1].strip())
|
|
isPOSTParam = True
|
|
break
|
|
|
|
if isPOSTParam:
|
|
continue
|
|
|
|
# time parameters not allowed in POST header
|
|
for p in self.TimeParams:
|
|
if p == key:
|
|
raise ValueError(
|
|
f"time parameter in line {nLine} not allowed in POST "
|
|
"request"
|
|
)
|
|
|
|
# stream parameters not allowed in POST header
|
|
for p in self.StreamParams:
|
|
if p == key:
|
|
raise ValueError(
|
|
f"stream parameter in line {nLine} not allowed in POST "
|
|
"request"
|
|
)
|
|
|
|
raise ValueError(f"invalid parameter in line {nLine}")
|
|
|
|
# stream parameters
|
|
toks = line.split()
|
|
nToks = len(toks)
|
|
if nToks not in (5, 6):
|
|
raise ValueError("invalid number of stream components in line {nLine}")
|
|
|
|
ro = RequestOptions()
|
|
|
|
# net, sta, loc, cha
|
|
ro.channel = RequestOptions.Channel()
|
|
ro.channel.net = toks[0].split(",")
|
|
ro.channel.sta = toks[1].split(",")
|
|
ro.channel.loc = toks[2].split(",")
|
|
ro.channel.cha = toks[3].split(",")
|
|
|
|
msg = "invalid %s value in line %i"
|
|
for net in ro.channel.net:
|
|
if ro.ChannelChars(net):
|
|
raise ValueError(msg % ("network", nLine))
|
|
for sta in ro.channel.sta:
|
|
if ro.ChannelChars(sta):
|
|
raise ValueError(msg % ("station", nLine))
|
|
for loc in ro.channel.loc:
|
|
if loc != "--" and ro.ChannelChars(loc):
|
|
raise ValueError(msg % ("location", nLine))
|
|
for cha in ro.channel.cha:
|
|
if ro.ChannelChars(cha):
|
|
raise ValueError(msg % ("channel", nLine))
|
|
|
|
# start/end time
|
|
ro.time = RequestOptions.Time()
|
|
ro.time.start = Time.FromString(toks[4])
|
|
logEnd = "-"
|
|
if len(toks) > 5:
|
|
ro.time.end = Time.FromString(toks[5])
|
|
logEnd = ro.time.end.iso()
|
|
|
|
seiscomp.logging.debug(
|
|
f"ro: {ro.channel.net}.{ro.channel.sta}.{ro.channel.loc}."
|
|
f"{ro.channel.cha} {ro.time.start.iso()} {logEnd}"
|
|
)
|
|
self.streams.append(ro)
|
|
|
|
if not self.streams:
|
|
raise ValueError("at least one stream line is required")
|
|
|
|
|
|
# vim: ts=4 et
|