You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
617 lines
22 KiB
Python
617 lines
22 KiB
Python
################################################################################
|
|
# Copyright (C) 2013-2014 gempa GmbH
|
|
#
|
|
# RequestOptions -- HTTP GET request parameters
|
|
#
|
|
# Author: Stephan Herrnkind
|
|
# Email: herrnkind@gempa.de
|
|
################################################################################
|
|
|
|
from __future__ import absolute_import, division, print_function
|
|
|
|
import fnmatch
|
|
import math
|
|
import re
|
|
|
|
from twisted.web import http
|
|
|
|
from seiscomp.core import Time
|
|
import seiscomp.logging
|
|
import seiscomp.math
|
|
|
|
from .utils import py3ustr, py3ustrlist
|
|
|
|
|
|
class RequestOptions:
|
|
|
|
# the match() method matched only patterns at the beginning of a string,
|
|
# since we have to ensure that no invalid character is present we use the
|
|
# search() method in combination with a negated pattern instead
|
|
FloatChars = re.compile(r'[^-0-9.]').search
|
|
ChannelChars = re.compile(r'[^A-Za-z0-9*?]').search
|
|
ChannelExtChars = re.compile(r'[^A-Za-z0-9*?+\-_]').search
|
|
TimeFormats = ['%FT%T.%f', # YYYY-MM-DDThh:mm:ss.ssssss
|
|
'%Y-%jT%T.%f', # YYYY-DDDThh:mm:ss.ssssss
|
|
'%FT%T', # YYYY-MM-DDThh:mm:ss
|
|
'%Y-%jT%T', # YYYY-DDDThh:mm:ss
|
|
'%FT%R', # YYYY-MM-DDThh:mm
|
|
'%Y-%jT%R', # YYYY-DDDThh:mm
|
|
'%FT%H', # YYYY-MM-DDThh
|
|
'%Y-%jT%H', # YYYY-DDDThh
|
|
'%F', # YYYY-MM-DD
|
|
'%Y-%j', # YYYY-DDD
|
|
'%Y', # YYYY
|
|
]
|
|
BooleanTrueValues = ['1', 'true', 't', 'yes', 'y']
|
|
BooleanFalseValues = ['0', 'false', 'f', 'no', 'n']
|
|
OutputFormats = [] # override in derived classes
|
|
|
|
PStart = ['starttime', 'start']
|
|
PEnd = ['endtime', 'end']
|
|
PStartBefore = ['startbefore']
|
|
PStartAfter = ['startafter']
|
|
PEndBefore = ['endbefore']
|
|
PEndAfter = ['endafter']
|
|
SimpleTimeParams = PStart + PEnd
|
|
WindowTimeParams = PStartBefore + PStartAfter + PEndBefore + PEndAfter
|
|
TimeParams = SimpleTimeParams + WindowTimeParams
|
|
|
|
PNet = ['network', 'net']
|
|
PSta = ['station', 'sta']
|
|
PLoc = ['location', 'loc']
|
|
PCha = ['channel', 'cha']
|
|
StreamParams = PNet + PSta + PLoc + PCha
|
|
|
|
PMinLat = ['minlatitude', 'minlat']
|
|
PMaxLat = ['maxlatitude', 'maxlat']
|
|
PMinLon = ['minlongitude', 'minlon']
|
|
PMaxLon = ['maxlongitude', 'maxlon']
|
|
PLat = ['latitude', 'lat']
|
|
PLon = ['longitude', 'lon']
|
|
PMinRadius = ['minradius']
|
|
PMaxRadius = ['maxradius']
|
|
GeoRectParams = PMinLat + PMaxLat + PMinLon + PMaxLon
|
|
GeoCircleParams = PLat + PLon + PMinRadius + PMaxRadius
|
|
GeoParams = GeoRectParams + GeoCircleParams
|
|
|
|
PFormat = ['format']
|
|
PNoData = ['nodata']
|
|
OutputParams = PFormat + PNoData
|
|
|
|
POSTParams = OutputParams
|
|
GETParams = StreamParams + SimpleTimeParams
|
|
|
|
#---------------------------------------------------------------------------
|
|
class Channel:
|
|
def __init__(self):
|
|
self.net = None
|
|
self.sta = None
|
|
self.loc = None
|
|
self.cha = None
|
|
|
|
def matchNet(self, value):
|
|
return self.match(value, self.net)
|
|
|
|
def matchSta(self, value):
|
|
return self.match(value, self.sta)
|
|
|
|
def matchLoc(self, value):
|
|
return self.match(value, self.loc, True)
|
|
|
|
def matchCha(self, value):
|
|
return self.match(value, self.cha)
|
|
|
|
@staticmethod
|
|
def match(value, globList, testEmpty=False):
|
|
if not globList:
|
|
return True
|
|
|
|
for glob in globList:
|
|
if testEmpty and value == '' and glob == '--':
|
|
return True
|
|
if fnmatch.fnmatchcase(value, glob):
|
|
return True
|
|
|
|
return False
|
|
|
|
#---------------------------------------------------------------------------
|
|
class Time:
|
|
|
|
def __init__(self):
|
|
self.simpleTime = True
|
|
self.start = None
|
|
self.end = None
|
|
# window time only
|
|
self.startBefore = None
|
|
self.startAfter = None
|
|
self.endBefore = None
|
|
self.endAfter = None
|
|
|
|
# used by FDSN Station and DataSelect
|
|
def match(self, start, end=None):
|
|
# simple time: limit to epochs intersecting with the specified time
|
|
# range
|
|
res = (self.start is None or end is None or end >= self.start) and \
|
|
(self.end is None or start <= self.end)
|
|
|
|
# window time: limit to epochs strictly starting or ending before or
|
|
# after a specified time value
|
|
if not self.simpleTime:
|
|
res = res and \
|
|
(self.startBefore is None or (
|
|
start is not None and start < self.startBefore)) and \
|
|
(self.startAfter is None or (
|
|
start is not None and start > self.startAfter)) and \
|
|
(self.endBefore is None or (
|
|
end is not None and end < self.endBefore)) and \
|
|
(self.endAfter is None or end is None or end > self.endAfter)
|
|
|
|
return res
|
|
|
|
#---------------------------------------------------------------------------
|
|
class Geo:
|
|
|
|
#-----------------------------------------------------------------------
|
|
class BBox:
|
|
|
|
def __init__(self):
|
|
self.minLat = None
|
|
self.maxLat = None
|
|
self.minLon = None
|
|
self.maxLon = None
|
|
|
|
def dateLineCrossing(self):
|
|
return self.minLon and self.maxLon and self.minLon > self.maxLon
|
|
|
|
#-----------------------------------------------------------------------
|
|
class BCircle:
|
|
|
|
def __init__(self):
|
|
self.lat = None
|
|
self.lon = None
|
|
self.minRad = None
|
|
self.maxRad = None
|
|
|
|
#-------------------------------------------------------------------
|
|
# Calculates outer bounding box
|
|
def calculateBBox(self):
|
|
def rad(degree):
|
|
return math.radians(degree)
|
|
|
|
def deg(radians):
|
|
return math.degrees(radians)
|
|
|
|
b = RequestOptions.Geo.BBox()
|
|
if self.maxRad is None or self.maxRad >= 180:
|
|
return b
|
|
|
|
b.minLat = self.lat - self.maxRad
|
|
b.maxLat = self.lat + self.maxRad
|
|
if b.minLat > -90 and b.maxLat < 90:
|
|
dLon = deg(math.asin(math.sin(rad(self.maxRad) /
|
|
math.cos(rad(self.lat)))))
|
|
b.minLon = self.lon - dLon
|
|
if b.minLon < -180:
|
|
b.minLon += 360
|
|
b.maxLon = self.lon + dLon
|
|
if b.maxLon > 180:
|
|
b.maxLon -= 360
|
|
else:
|
|
# pole within distance: one latitude and no longitude
|
|
# restrictions remains
|
|
if b.minLat <= -90:
|
|
b.minLat = None
|
|
else:
|
|
b.maxLat = None
|
|
b.minLon = None
|
|
b.maxLon = None
|
|
|
|
return b
|
|
|
|
#-----------------------------------------------------------------------
|
|
def __init__(self):
|
|
self.bBox = None
|
|
self.bCircle = None
|
|
|
|
#-----------------------------------------------------------------------
|
|
def match(self, lat, lon):
|
|
if self.bBox is not None:
|
|
b = self.bBox
|
|
if b.minLat is not None and lat < b.minLat:
|
|
return False
|
|
if b.maxLat is not None and lat > b.maxLat:
|
|
return False
|
|
# date line crossing if minLon > maxLon
|
|
if b.dateLineCrossing():
|
|
return lon >= b.minLon or lon <= b.maxLon
|
|
if b.minLon is not None and lon < b.minLon:
|
|
return False
|
|
if b.maxLon is not None and lon > b.maxLon:
|
|
return False
|
|
return True
|
|
|
|
if self.bCircle:
|
|
c = self.bCircle
|
|
dist = seiscomp.math.delazi(c.lat, c.lon, lat, lon)
|
|
if c.minRad is not None and dist[0] < c.minRad:
|
|
return False
|
|
if c.maxRad is not None and dist[0] > c.maxRad:
|
|
return False
|
|
return True
|
|
|
|
return False
|
|
|
|
#---------------------------------------------------------------------------
|
|
def __init__(self):
|
|
self.service = ""
|
|
self.accessTime = Time.GMT()
|
|
self.userName = None
|
|
|
|
self.time = None
|
|
self.channel = None
|
|
self.geo = None
|
|
|
|
self.noData = http.NO_CONTENT
|
|
self.format = None
|
|
|
|
self._args = {}
|
|
self.streams = [] # 1 entry for GET, multipl
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseOutput(self):
|
|
# nodata
|
|
code = self.parseInt(self.PNoData)
|
|
if code is not None:
|
|
if code not in (http.NO_CONTENT, http.NOT_FOUND):
|
|
self.raiseValueError(self.PNoData[0])
|
|
self.noData = code
|
|
|
|
# format
|
|
key, value = self.getFirstValue(self.PFormat)
|
|
if value is None:
|
|
# no format specified: default to first in list if available
|
|
if len(self.OutputFormats) > 0:
|
|
self.format = self.OutputFormats[0]
|
|
else:
|
|
value = value.lower()
|
|
if value in self.OutputFormats:
|
|
self.format = value
|
|
else:
|
|
self.raiseValueError(key)
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseChannel(self):
|
|
c = RequestOptions.Channel()
|
|
|
|
c.net = self.parseChannelChars(self.PNet, False, True)
|
|
c.sta = self.parseChannelChars(self.PSta)
|
|
c.loc = self.parseChannelChars(self.PLoc, True)
|
|
c.cha = self.parseChannelChars(self.PCha)
|
|
|
|
if c.net or c.sta or c.loc or c.cha:
|
|
self.channel = c
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseTime(self, parseWindowTime=False):
|
|
t = RequestOptions.Time()
|
|
|
|
# start[time], end[time]
|
|
t.start = self.parseTimeStr(self.PStart)
|
|
t.end = self.parseTimeStr(self.PEnd)
|
|
|
|
simpleTime = t.start is not None or t.end is not None
|
|
|
|
# [start,end][before,after]
|
|
if parseWindowTime:
|
|
t.startBefore = self.parseTimeStr(self.PStartBefore)
|
|
t.startAfter = self.parseTimeStr(self.PStartAfter)
|
|
t.endBefore = self.parseTimeStr(self.PEndBefore)
|
|
t.endAfter = self.parseTimeStr(self.PEndAfter)
|
|
|
|
windowTime = t.startBefore is not None or t.startAfter is not None or \
|
|
t.endBefore is not None or t.endAfter is not None
|
|
if simpleTime or windowTime:
|
|
self.time = t
|
|
self.time.simpleTime = not windowTime
|
|
|
|
elif simpleTime:
|
|
self.time = t
|
|
self.time.simpleTime = True
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseGeo(self):
|
|
# bounding box (optional)
|
|
b = RequestOptions.Geo.BBox()
|
|
b.minLat = self.parseFloat(self.PMinLat, -90, 90)
|
|
b.maxLat = self.parseFloat(self.PMaxLat, -90, 90)
|
|
if b.minLat is not None and b.maxLat is not None and \
|
|
b.minLat > b.maxLat:
|
|
raise ValueError("%s exceeds %s" % (self.PMinLat[0],
|
|
self.PMaxLat[0]))
|
|
|
|
b.minLon = self.parseFloat(self.PMinLon, -180, 180)
|
|
b.maxLon = self.parseFloat(self.PMaxLon, -180, 180)
|
|
# maxLon < minLon -> date line crossing
|
|
|
|
hasBBoxParam = b.minLat is not None or b.maxLat is not None or \
|
|
b.minLon is not None or b.maxLon is not None
|
|
|
|
# bounding circle (optional)
|
|
c = RequestOptions.Geo.BCircle()
|
|
c.lat = self.parseFloat(self.PLat, -90, 90)
|
|
c.lon = self.parseFloat(self.PLon, -180, 180)
|
|
c.minRad = self.parseFloat(self.PMinRadius, 0, 180)
|
|
c.maxRad = self.parseFloat(self.PMaxRadius, 0, 180)
|
|
if c.minRad is not None and c.maxRad is not None and \
|
|
c.minRad > c.maxRad:
|
|
raise ValueError("%s exceeds %s" % (self.PMinRadius[0],
|
|
self.PMaxRadius[0]))
|
|
|
|
hasBCircleRadParam = c.minRad is not None or c.maxRad is not None
|
|
hasBCircleParam = c.lat is not None or c.lon is not None or \
|
|
hasBCircleRadParam
|
|
|
|
# bounding box and bounding circle may not be combined
|
|
if hasBBoxParam and hasBCircleParam:
|
|
raise ValueError("bounding box and bounding circle parameters " \
|
|
"may not be combined")
|
|
if hasBBoxParam:
|
|
self.geo = RequestOptions.Geo()
|
|
self.geo.bBox = b
|
|
elif hasBCircleRadParam:
|
|
self.geo = RequestOptions.Geo()
|
|
if c.lat is None:
|
|
c.lat = .0
|
|
if c.lon is None:
|
|
c.lon = .0
|
|
self.geo.bCircle = c
|
|
|
|
#---------------------------------------------------------------------------
|
|
@staticmethod
|
|
def _assertValueRange(key, v, minValue, maxValue):
|
|
if (minValue is not None and v < minValue) or \
|
|
(maxValue is not None and v > maxValue):
|
|
minStr, maxStr = '-inf', 'inf'
|
|
if minValue is not None:
|
|
minStr = str(minValue)
|
|
if maxValue is not None:
|
|
maxStr = str(maxValue)
|
|
raise ValueError(
|
|
"parameter not in domain [%s,%s]: %s" % (minStr, maxStr, key))
|
|
|
|
#---------------------------------------------------------------------------
|
|
@staticmethod
|
|
def raiseValueError(key):
|
|
raise ValueError("invalid value in parameter: %s" % key)
|
|
|
|
#---------------------------------------------------------------------------
|
|
def getFirstValue(self, keys):
|
|
for key in keys:
|
|
if key in self._args:
|
|
return key, self._args[key][0].strip()
|
|
|
|
return None, None
|
|
|
|
#---------------------------------------------------------------------------
|
|
def getValues(self, keys):
|
|
v = []
|
|
for key in keys:
|
|
if key in self._args:
|
|
v += self._args[key]
|
|
return v
|
|
|
|
#---------------------------------------------------------------------------
|
|
def getListValues(self, keys, lower=False):
|
|
values = set()
|
|
for key in keys:
|
|
if not key in self._args:
|
|
continue
|
|
for vList in self._args[key]:
|
|
for v in vList.split(','):
|
|
if v is None:
|
|
continue
|
|
v = v.strip()
|
|
if lower:
|
|
v = v.lower()
|
|
values.add(v)
|
|
|
|
return values
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseInt(self, keys, minValue=None, maxValue=None):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
try:
|
|
i = int(value)
|
|
except ValueError:
|
|
raise ValueError("invalid integer value in parameter: %s" % key)
|
|
self._assertValueRange(key, i, minValue, maxValue)
|
|
return i
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseFloat(self, keys, minValue=None, maxValue=None):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
if self.FloatChars(value):
|
|
raise ValueError("invalid characters in float parameter: %s " \
|
|
"(scientific notation forbidden by spec)" % key)
|
|
|
|
try:
|
|
f = float(value)
|
|
except ValueError:
|
|
raise ValueError("invalid float value in parameter: %s" % key)
|
|
self._assertValueRange(key, f, minValue, maxValue)
|
|
return f
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseBool(self, keys):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
value = value.lower()
|
|
if value in self.BooleanTrueValues:
|
|
return True
|
|
if value in self.BooleanFalseValues:
|
|
return False
|
|
|
|
raise ValueError("invalid boolean value in parameter: %s" % key)
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseTimeStr(self, keys):
|
|
key, value = self.getFirstValue(keys)
|
|
|
|
if value is None:
|
|
return None
|
|
|
|
time = Time()
|
|
timeValid = False
|
|
for fmt in RequestOptions.TimeFormats:
|
|
if time.fromString(value, fmt):
|
|
timeValid = True
|
|
break
|
|
|
|
if not timeValid:
|
|
raise ValueError("invalid date format in parameter: %s" % key)
|
|
|
|
return time
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseChannelChars(self, keys, allowEmpty=False, useExtChars=False):
|
|
# channel parameters may be specified as a comma separated list and may
|
|
# be repeated several times
|
|
values = None
|
|
for vList in self.getValues(keys):
|
|
if values is None:
|
|
values = []
|
|
for v in vList.split(','):
|
|
v = v.strip()
|
|
if allowEmpty and (v == '--' or len(v) == 0):
|
|
values.append('--')
|
|
continue
|
|
|
|
if (useExtChars and self.ChannelExtChars(v)) or \
|
|
(not useExtChars and self.ChannelChars(v)):
|
|
raise ValueError("invalid characters in parameter: " \
|
|
"%s" % keys[0])
|
|
values.append(v)
|
|
|
|
return values
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parseGET(self, args):
|
|
# transform keys to lower case
|
|
if args is not None:
|
|
for k, v in args.items():
|
|
k = py3ustr(k.lower())
|
|
if k not in self.GETParams:
|
|
raise ValueError("invalid param: %s" % k)
|
|
|
|
self._args[k] = py3ustrlist(v)
|
|
|
|
#---------------------------------------------------------------------------
|
|
def parsePOST(self, content):
|
|
nLine = 0
|
|
|
|
for line in content:
|
|
nLine += 1
|
|
line = py3ustr(line.strip())
|
|
|
|
# ignore empty and comment lines
|
|
if len(line) == 0 or line[0] == '#':
|
|
continue
|
|
|
|
# collect parameter (non stream lines)
|
|
toks = line.split("=", 1)
|
|
if len(toks) > 1:
|
|
key = toks[0].strip().lower()
|
|
|
|
isPOSTParam = False
|
|
for p in self.POSTParams:
|
|
if p == key:
|
|
if key not in self._args:
|
|
self._args[key] = []
|
|
self._args[key].append(toks[1].strip())
|
|
isPOSTParam = True
|
|
break
|
|
|
|
if isPOSTParam:
|
|
continue
|
|
|
|
# time parameters not allowed in POST header
|
|
for p in self.TimeParams:
|
|
if p == key:
|
|
raise ValueError("time parameter in line %i not " \
|
|
"allowed in POST request" % nLine)
|
|
|
|
# stream parameters not allowed in POST header
|
|
for p in self.StreamParams:
|
|
if p == key:
|
|
raise ValueError("stream parameter in line %i not " \
|
|
"allowed in POST request" % nLine)
|
|
|
|
raise ValueError("invalid parameter in line %i" % nLine)
|
|
|
|
|
|
# stream parameters
|
|
toks = line.split()
|
|
nToks = len(toks)
|
|
if nToks not in (5, 6):
|
|
raise ValueError("invalid number of stream components " \
|
|
"in line %i" % nLine)
|
|
|
|
ro = RequestOptions()
|
|
|
|
# net, sta, loc, cha
|
|
ro.channel = RequestOptions.Channel()
|
|
ro.channel.net = toks[0].split(',')
|
|
ro.channel.sta = toks[1].split(',')
|
|
ro.channel.loc = toks[2].split(',')
|
|
ro.channel.cha = toks[3].split(',')
|
|
|
|
msg = "invalid %s value in line %i"
|
|
for net in ro.channel.net:
|
|
if ro.ChannelChars(net):
|
|
raise ValueError(msg % ('network', nLine))
|
|
for sta in ro.channel.sta:
|
|
if ro.ChannelChars(sta):
|
|
raise ValueError(msg % ('station', nLine))
|
|
for loc in ro.channel.loc:
|
|
if loc != "--" and ro.ChannelChars(loc):
|
|
raise ValueError(msg % ('location', nLine))
|
|
for cha in ro.channel.cha:
|
|
if ro.ChannelChars(cha):
|
|
raise ValueError(msg % ('channel', nLine))
|
|
|
|
# start/end time
|
|
ro.time = RequestOptions.Time()
|
|
ro.time.start = Time()
|
|
for fmt in RequestOptions.TimeFormats:
|
|
if ro.time.start.fromString(toks[4], fmt):
|
|
break
|
|
logEnd = "-"
|
|
if len(toks) > 5:
|
|
ro.time.end = Time()
|
|
for fmt in RequestOptions.TimeFormats:
|
|
if ro.time.end.fromString(toks[5], fmt):
|
|
break
|
|
logEnd = ro.time.end.iso()
|
|
|
|
seiscomp.logging.debug("ro: %s.%s.%s.%s %s %s" % (
|
|
ro.channel.net, ro.channel.sta, ro.channel.loc,
|
|
ro.channel.cha, ro.time.start.iso(), logEnd))
|
|
self.streams.append(ro)
|
|
|
|
if len(self.streams) == 0:
|
|
raise ValueError("at least one stream line is required")
|
|
|
|
|
|
# vim: ts=4 et
|