[seiscomp, scanloc] Install, add .gitignore

lib/python/seiscomp/fdsnws/authresource.py

@@ -0,0 +1,85 @@
+################################################################################
+# Copyright (C) 2013-2014 by gempa GmbH
+#
+# HTTP -- Utility methods which generate HTTP result strings
+#
+# Author:  Stephan Herrnkind
+# Email:   herrnkind@gempa.de
+################################################################################
+
+import base64
+import datetime
+import hashlib
+import json
+import time
+import dateutil.parser
+
+from twisted.web import http
+
+import gnupg
+
+import seiscomp.logging
+
+from .utils import accessLog, u_str
+
+from .http import BaseResource
+
+
+################################################################################
+class AuthResource(BaseResource):
+    isLeaf = True
+
+    def __init__(self, version, gnupghome, userdb):
+        super().__init__(version)
+
+        self.__gpg = gnupg.GPG(gnupghome=gnupghome)
+        self.__userdb = userdb
+
+    # ---------------------------------------------------------------------------
+    def render_POST(self, request):
+        request.setHeader("Content-Type", "text/plain; charset=utf-8")
+
+        try:
+            verified = self.__gpg.decrypt(request.content.getvalue())
+
+        except OSError as e:
+            msg = "gpg decrypt error"
+            seiscomp.logging.warning(f"{msg}: {e}")
+            return self.renderErrorPage(request, http.INTERNAL_SERVER_ERROR, msg)
+
+        except Exception as e:
+            msg = "invalid token"
+            seiscomp.logging.warning(f"{msg}: {e}")
+            return self.renderErrorPage(request, http.BAD_REQUEST, msg)
+
+        if verified.trust_level is None or verified.trust_level < verified.TRUST_FULLY:
+            msg = "token has invalid signature"
+            seiscomp.logging.warning(msg)
+            return self.renderErrorPage(request, http.BAD_REQUEST, msg)
+
+        try:
+            attributes = json.loads(u_str(verified.data))
+            td = dateutil.parser.parse(
+                attributes["valid_until"]
+            ) - datetime.datetime.now(dateutil.tz.tzutc())
+            lifetime = td.seconds + td.days * 24 * 3600
+
+        except Exception as e:
+            msg = "token has invalid validity"
+            seiscomp.logging.warning(f"{msg}: {e}")
+            return self.renderErrorPage(request, http.BAD_REQUEST, msg)
+
+        if lifetime <= 0:
+            msg = "token is expired"
+            seiscomp.logging.warning(msg)
+            return self.renderErrorPage(request, http.BAD_REQUEST, msg)
+
+        userid = base64.urlsafe_b64encode(hashlib.sha256(verified.data).digest()[:18])
+        password = self.__userdb.addUser(
+            u_str(userid),
+            attributes,
+            time.time() + min(lifetime, 24 * 3600),
+            u_str(verified.data),
+        )
+        accessLog(request, None, http.OK, len(userid) + len(password) + 1, None)
+        return userid + b":" + password

lib/python/seiscomp/fdsnws/dataselect.py

@@ -0,0 +1,796 @@
+################################################################################
+# Copyright (C) 2013-2014 by gempa GmbH
+#
+# FDSNDataSelect -- Implements the fdsnws-dataselect Web service, see
+#   http://www.fdsn.org/webservices/
+#
+# Feature notes:
+#   - 'quality' request parameter not implemented (information not available in
+#     SeisComP)
+#   - 'minimumlength' parameter is not implemented
+#   - 'longestonly' parameter is not implemented
+#
+# Author:  Stephan Herrnkind
+# Email:   herrnkind@gempa.de
+################################################################################
+
+import time
+
+from io import BytesIO
+
+import dateutil.parser
+
+from twisted.cred import portal
+from twisted.web import http, resource, server
+from twisted.internet import interfaces, reactor
+
+from zope.interface import implementer
+
+from seiscomp import logging, mseedlite
+
+from seiscomp.client import Application
+from seiscomp.core import Array, Record, Time
+from seiscomp.io import RecordInput, RecordStream
+
+from .http import HTTP, BaseResource
+from .request import RequestOptions
+from . import utils
+
+from .reqtrack import RequestTrackerDB
+from .fastsds import SDS
+
+VERSION = "1.1.3"
+
+################################################################################
+
+
+class _DataSelectRequestOptions(RequestOptions):
+    MinTime = Time(0, 1)
+
+    PQuality = ["quality"]
+    PMinimumLength = ["minimumlength"]
+    PLongestOnly = ["longestonly"]
+
+    QualityValues = ["B", "D", "M", "Q", "R"]
+    OutputFormats = ["miniseed", "mseed"]
+
+    POSTParams = RequestOptions.POSTParams + PQuality + PMinimumLength + PLongestOnly
+    GETParams = RequestOptions.GETParams + POSTParams
+
+    # ---------------------------------------------------------------------------
+    def __init__(self):
+        super().__init__()
+
+        self.service = "fdsnws-dataselect"
+
+        self.quality = self.QualityValues[0]
+        self.minimumLength = None
+        self.longestOnly = None
+
+    # ---------------------------------------------------------------------------
+    def _checkTimes(self, realtimeGap):
+        maxEndTime = Time(self.accessTime)
+        if realtimeGap is not None:
+            maxEndTime -= Time(realtimeGap, 0)
+
+        for ro in self.streams:
+            # create time if non was specified
+            if ro.time is None:
+                ro.time = RequestOptions.Time()
+            # restrict time to 1970 - now
+            if ro.time.start is None or ro.time.start < self.MinTime:
+                ro.time.start = self.MinTime
+            if ro.time.end is None or ro.time.end > maxEndTime:
+                ro.time.end = maxEndTime
+
+        # remove items with start time >= end time
+        self.streams = [x for x in self.streams if x.time.start < x.time.end]
+
+    # ---------------------------------------------------------------------------
+    def parse(self):
+        # quality (optional), currently not supported
+        key, value = self.getFirstValue(self.PQuality)
+        if value is not None:
+            value = value.upper()
+            if value in self.QualityValues:
+                self.quality = value
+            else:
+                self.raiseValueError(key)
+
+        # minimumlength(optional), currently not supported
+        self.minimumLength = self.parseFloat(self.PMinimumLength, 0)
+
+        # longestonly (optional), currently not supported
+        self.longestOnly = self.parseBool(self.PLongestOnly)
+
+        # generic parameters
+        self.parseTime()
+        self.parseChannel()
+        self.parseOutput()
+
+
+################################################################################
+class _MyRecordStream:
+    def __init__(self, url, trackerList, bufferSize):
+        self.__url = url
+        self.__trackerList = trackerList
+        self.__bufferSize = bufferSize
+        self.__tw = []
+
+    def addStream(self, net, sta, loc, cha, startt, endt, restricted, archNet):
+        self.__tw.append((net, sta, loc, cha, startt, endt, restricted, archNet))
+
+    @staticmethod
+    def __override_network(data, net):
+        inp = BytesIO(data)
+        out = BytesIO()
+
+        for rec in mseedlite.Input(inp):
+            rec.net = net
+            rec_len_exp = 9
+
+            while (1 << rec_len_exp) < rec.size:
+                rec_len_exp += 1
+
+            rec.write(out, rec_len_exp)
+
+        return out.getvalue()
+
+    def input(self):
+        fastsdsPrefix = "fastsds://"
+
+        if self.__url.startswith(fastsdsPrefix):
+            fastsds = SDS(self.__url[len(fastsdsPrefix) :])
+
+        else:
+            fastsds = None
+
+        for net, sta, loc, cha, startt, endt, restricted, archNet in self.__tw:
+            if not archNet:
+                archNet = net
+
+            size = 0
+
+            if fastsds:
+                start = dateutil.parser.parse(startt.iso()).replace(tzinfo=None)
+                end = dateutil.parser.parse(endt.iso()).replace(tzinfo=None)
+
+                for data in fastsds.getRawBytes(
+                    start, end, archNet, sta, loc, cha, self.__bufferSize
+                ):
+                    size += len(data)
+
+                    if archNet == net:
+                        yield data
+
+                    else:
+                        try:
+                            yield self.__override_network(data, net)
+
+                        except Exception as e:
+                            logging.error(f"could not override network code: {e}")
+
+            else:
+                rs = RecordStream.Open(self.__url)
+
+                if rs is None:
+                    logging.error("could not open record stream")
+                    break
+
+                rs.addStream(archNet, sta, loc, cha, startt, endt)
+                rsInput = RecordInput(rs, Array.INT, Record.SAVE_RAW)
+                eof = False
+
+                while not eof:
+                    data = b""
+
+                    while len(data) < self.__bufferSize:
+                        try:
+                            rec = rsInput.next()
+
+                        except Exception as e:
+                            logging.error(str(e))
+                            eof = True
+                            break
+
+                        if rec is None:
+                            eof = True
+                            break
+
+                        data += rec.raw().str()
+
+                    if data:
+                        size += len(data)
+
+                        if archNet == net:
+                            yield data
+
+                        else:
+                            try:
+                                yield self.__override_network(data, net)
+
+                            except Exception as e:
+                                logging.error(f"could not override network code: {e}")
+
+            for tracker in self.__trackerList:
+                net_class = "t" if net[0] in "0123456789XYZ" else "p"
+
+                if size == 0:
+                    tracker.line_status(
+                        startt,
+                        endt,
+                        net,
+                        sta,
+                        cha,
+                        loc,
+                        restricted,
+                        net_class,
+                        True,
+                        [],
+                        "fdsnws",
+                        "NODATA",
+                        0,
+                        "",
+                    )
+
+                else:
+                    tracker.line_status(
+                        startt,
+                        endt,
+                        net,
+                        sta,
+                        cha,
+                        loc,
+                        restricted,
+                        net_class,
+                        True,
+                        [],
+                        "fdsnws",
+                        "OK",
+                        size,
+                        "",
+                    )
+
+
+################################################################################
+@implementer(interfaces.IPushProducer)
+class _WaveformProducer:
+    def __init__(self, req, ro, rs, fileName, trackerList):
+        self.req = req
+        self.ro = ro
+        self.it = rs.input()
+
+        self.fileName = fileName
+        self.written = 0
+
+        self.trackerList = trackerList
+        self.paused = False
+        self.stopped = False
+        self.running = False
+
+    def _flush(self, data):
+        if self.stopped:
+            return
+
+        if not self.paused:
+            reactor.callInThread(self._collectData)
+
+        else:
+            self.running = False
+
+        if self.written == 0:
+            self.req.setHeader("Content-Type", "application/vnd.fdsn.mseed")
+            self.req.setHeader(
+                "Content-Disposition", f"attachment; filename={self.fileName}"
+            )
+
+        self.req.write(data)
+        self.written += len(data)
+
+    def _finish(self):
+        if self.stopped:
+            return
+
+        if self.written == 0:
+            msg = "no waveform data found"
+            errorpage = HTTP.renderErrorPage(
+                self.req, http.NO_CONTENT, msg, VERSION, self.ro
+            )
+
+            if errorpage:
+                self.req.write(errorpage)
+
+            for tracker in self.trackerList:
+                tracker.volume_status("fdsnws", "NODATA", 0, "")
+                tracker.request_status("END", "")
+
+        else:
+            logging.debug(
+                f"{self.ro.service}: returned {self.written} bytes of mseed data"
+            )
+            utils.accessLog(self.req, self.ro, http.OK, self.written, None)
+
+            for tracker in self.trackerList:
+                tracker.volume_status("fdsnws", "OK", self.written, "")
+                tracker.request_status("END", "")
+
+        self.req.unregisterProducer()
+        self.req.finish()
+
+    def _collectData(self):
+        try:
+            reactor.callFromThread(self._flush, next(self.it))
+
+        except StopIteration:
+            reactor.callFromThread(self._finish)
+
+    def pauseProducing(self):
+        self.paused = True
+
+    def resumeProducing(self):
+        self.paused = False
+
+        if not self.running:
+            self.running = True
+            reactor.callInThread(self._collectData)
+
+    def stopProducing(self):
+        self.stopped = True
+
+        logging.debug(
+            f"{self.ro.service}: returned {self.written} bytes of mseed data (not "
+            "completed)"
+        )
+        utils.accessLog(self.req, self.ro, http.OK, self.written, "not completed")
+
+        for tracker in self.trackerList:
+            tracker.volume_status("fdsnws", "ERROR", self.written, "")
+            tracker.request_status("END", "")
+
+        self.req.unregisterProducer()
+        self.req.finish()
+
+
+################################################################################
+@implementer(portal.IRealm)
+class FDSNDataSelectRealm:
+    # ---------------------------------------------------------------------------
+    def __init__(self, inv, bufferSize, access):
+        self.__inv = inv
+        self.__bufferSize = bufferSize
+        self.__access = access
+
+    # ---------------------------------------------------------------------------
+    def requestAvatar(self, avatarId, _mind, *interfaces_):
+        if resource.IResource in interfaces_:
+            return (
+                resource.IResource,
+                FDSNDataSelect(
+                    self.__inv,
+                    self.__bufferSize,
+                    self.__access,
+                    {"mail": utils.u_str(avatarId), "blacklisted": False},
+                ),
+                lambda: None,
+            )
+
+        raise NotImplementedError()
+
+
+################################################################################
+@implementer(portal.IRealm)
+class FDSNDataSelectAuthRealm:
+    # ---------------------------------------------------------------------------
+    def __init__(self, inv, bufferSize, access, userdb):
+        self.__inv = inv
+        self.__bufferSize = bufferSize
+        self.__access = access
+        self.__userdb = userdb
+
+    # ---------------------------------------------------------------------------
+    def requestAvatar(self, avatarId, _mind, *interfaces_):
+        if resource.IResource in interfaces_:
+            return (
+                resource.IResource,
+                FDSNDataSelect(
+                    self.__inv,
+                    self.__bufferSize,
+                    self.__access,
+                    self.__userdb.getAttributes(utils.u_str(avatarId)),
+                ),
+                lambda: None,
+            )
+
+        raise NotImplementedError()
+
+
+################################################################################
+class FDSNDataSelect(BaseResource):
+    isLeaf = True
+
+    # ---------------------------------------------------------------------------
+    def __init__(self, inv, bufferSize, access=None, user=None):
+        super().__init__(VERSION)
+
+        self._rsURL = Application.Instance().recordStreamURL()
+        self.__inv = inv
+        self.__access = access
+        self.__user = user
+        self.__bufferSize = bufferSize
+
+    # ---------------------------------------------------------------------------
+    def render_OPTIONS(self, req):
+        req.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+        req.setHeader(
+            "Access-Control-Allow-Headers",
+            "Accept, Content-Type, X-Requested-With, Origin",
+        )
+        req.setHeader("Content-Type", "text/plain; charset=utf-8")
+        return ""
+
+    # ---------------------------------------------------------------------------
+    def render_GET(self, req):
+        # Parse and validate GET parameters
+        ro = _DataSelectRequestOptions()
+        ro.userName = self.__user and self.__user.get("mail")
+        try:
+            ro.parseGET(req.args)
+            ro.parse()
+            # the GET operation supports exactly one stream filter
+            ro.streams.append(ro)
+        except ValueError as e:
+            logging.warning(str(e))
+            return self.renderErrorPage(req, http.BAD_REQUEST, str(e), ro)
+
+        return self._processRequest(req, ro)
+
+    # ---------------------------------------------------------------------------
+    def render_POST(self, req):
+        # Parse and validate POST parameters
+        ro = _DataSelectRequestOptions()
+        ro.userName = self.__user and self.__user.get("mail")
+        try:
+            ro.parsePOST(req.content)
+            ro.parse()
+        except ValueError as e:
+            logging.warning(str(e))
+            return self.renderErrorPage(req, http.BAD_REQUEST, str(e), ro)
+
+        return self._processRequest(req, ro)
+
+    # -----------------------------------------------------------------------
+    def _networkIter(self, ro):
+        for i in range(self.__inv.networkCount()):
+            net = self.__inv.network(i)
+
+            # network code
+            if ro.channel and not ro.channel.matchNet(net.code()):
+                continue
+
+            # start and end time
+            if ro.time:
+                try:
+                    end = net.end()
+                except ValueError:
+                    end = None
+                if not ro.time.match(net.start(), end):
+                    continue
+
+            yield net
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def _stationIter(net, ro):
+        for i in range(net.stationCount()):
+            sta = net.station(i)
+
+            # station code
+            if ro.channel and not ro.channel.matchSta(sta.code()):
+                continue
+
+            # start and end time
+            if ro.time:
+                try:
+                    end = sta.end()
+                except ValueError:
+                    end = None
+                if not ro.time.match(sta.start(), end):
+                    continue
+
+            yield sta
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def _locationIter(sta, ro):
+        for i in range(sta.sensorLocationCount()):
+            loc = sta.sensorLocation(i)
+
+            # location code
+            if ro.channel and not ro.channel.matchLoc(loc.code()):
+                continue
+
+            # start and end time
+            if ro.time:
+                try:
+                    end = loc.end()
+                except ValueError:
+                    end = None
+                if not ro.time.match(loc.start(), end):
+                    continue
+
+            yield loc
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def _streamIter(loc, ro):
+        for i in range(loc.streamCount()):
+            stream = loc.stream(i)
+
+            # stream code
+            if ro.channel and not ro.channel.matchCha(stream.code()):
+                continue
+
+            # start and end time
+            if ro.time:
+                try:
+                    end = stream.end()
+                except ValueError:
+                    end = None
+                if not ro.time.match(stream.start(), end):
+                    continue
+
+            yield stream, False
+
+        for i in range(loc.auxStreamCount()):
+            stream = loc.auxStream(i)
+
+            # stream code
+            if ro.channel and not ro.channel.matchCha(stream.code()):
+                continue
+
+            # start and end time
+            if ro.time:
+                try:
+                    end = stream.end()
+                except ValueError:
+                    end = None
+                if not ro.time.match(stream.start(), end):
+                    continue
+
+            yield stream, True
+
+    # ---------------------------------------------------------------------------
+    def _processRequest(self, req, ro):
+        # pylint: disable=W0212
+
+        if ro.quality not in ("B", "M"):
+            msg = "quality other than 'B' or 'M' not supported"
+            return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+
+        if ro.minimumLength:
+            msg = "enforcing of minimum record length not supported"
+            return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+
+        if ro.longestOnly:
+            msg = "limitation to longest segment not supported"
+            return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+
+        app = Application.Instance()
+        ro._checkTimes(app._realtimeGap)
+
+        maxSamples = None
+        if app._samplesM is not None:
+            maxSamples = app._samplesM * 1000000
+            samples = 0
+
+        trackerList = []
+        userIP = ""
+
+        if app._trackdbEnabled or app._requestLog:
+            xff = req.requestHeaders.getRawHeaders("x-forwarded-for")
+            if xff:
+                userIP = xff[0].split(",")[0].strip()
+            else:
+                userIP = req.getClientIP()
+
+            clientID = req.getHeader("User-Agent")
+            if clientID:
+                clientID = clientID[:80]
+            else:
+                clientID = "fdsnws"
+
+        if app._trackdbEnabled:
+            if ro.userName:
+                userID = ro.userName
+            else:
+                userID = app._trackdbDefaultUser
+
+            reqID = f"ws{str(int(round(time.time() * 1000) - 1420070400000))}"
+            tracker = RequestTrackerDB(
+                clientID,
+                app.connection(),
+                reqID,
+                "WAVEFORM",
+                userID,
+                f"REQUEST WAVEFORM {reqID}",
+                "fdsnws",
+                userIP,
+                req.getClientIP(),
+            )
+
+            trackerList.append(tracker)
+
+        if app._requestLog:
+            tracker = app._requestLog.tracker(ro.service, ro.userName, userIP, clientID)
+            trackerList.append(tracker)
+
+        # Open record stream
+        rs = _MyRecordStream(self._rsURL, trackerList, self.__bufferSize)
+
+        forbidden = None
+        auxStreamsFound = False
+
+        # Add request streams
+        # iterate over inventory networks
+        for s in ro.streams:
+            for net in self._networkIter(s):
+                netRestricted = utils.isRestricted(net)
+                if not trackerList and netRestricted and not self.__user:
+                    forbidden = forbidden or (forbidden is None)
+                    continue
+
+                for sta in self._stationIter(net, s):
+                    staRestricted = utils.isRestricted(sta)
+                    if not trackerList and staRestricted and not self.__user:
+                        forbidden = forbidden or (forbidden is None)
+                        continue
+
+                    for loc in self._locationIter(sta, s):
+                        for cha, aux in self._streamIter(loc, s):
+                            start_time = max(cha.start(), s.time.start)
+
+                            try:
+                                end_time = min(cha.end(), s.time.end)
+                            except ValueError:
+                                end_time = s.time.end
+
+                            streamRestricted = (
+                                netRestricted
+                                or staRestricted
+                                or utils.isRestricted(cha)
+                            )
+                            if streamRestricted and (
+                                not self.__user
+                                or (
+                                    self.__access
+                                    and not self.__access.authorize(
+                                        self.__user,
+                                        net.code(),
+                                        sta.code(),
+                                        loc.code(),
+                                        cha.code(),
+                                        start_time,
+                                        end_time,
+                                    )
+                                )
+                            ):
+                                for tracker in trackerList:
+                                    net_class = (
+                                        "t" if net.code()[0] in "0123456789XYZ" else "p"
+                                    )
+                                    tracker.line_status(
+                                        start_time,
+                                        end_time,
+                                        net.code(),
+                                        sta.code(),
+                                        cha.code(),
+                                        loc.code(),
+                                        True,
+                                        net_class,
+                                        True,
+                                        [],
+                                        "fdsnws",
+                                        "DENIED",
+                                        0,
+                                        "",
+                                    )
+
+                                forbidden = forbidden or (forbidden is None)
+                                continue
+
+                            forbidden = False
+
+                            # aux streams are deprecated, mark aux streams as
+                            # present to report warning later on, also do not
+                            # count aux stream samples due to their loose
+                            # binding to a aux device and source which only
+                            # optionally contains a sampling rate
+                            if aux:
+                                auxStreamsFound = True
+                            # enforce maximum sample per request restriction
+                            elif maxSamples is not None:
+                                try:
+                                    n = cha.sampleRateNumerator()
+                                    d = cha.sampleRateDenominator()
+                                except ValueError:
+                                    logging.warning(
+                                        "skipping stream without sampling rate "
+                                        f"definition: {net.code()}.{sta.code()}."
+                                        f"{loc.code()}.{cha.code()}"
+                                    )
+                                    continue
+
+                                # calculate number of samples for requested
+                                # time window
+                                diffSec = (end_time - start_time).length()
+                                samples += int(diffSec * n / d)
+                                if samples > maxSamples:
+                                    msg = (
+                                        f"maximum number of {app._samplesM}M samples "
+                                        "exceeded"
+                                    )
+                                    return self.renderErrorPage(
+                                        req, http.REQUEST_ENTITY_TOO_LARGE, msg, ro
+                                    )
+
+                            logging.debug(
+                                f"adding stream: {net.code()}.{sta.code()}.{loc.code()}"
+                                f".{cha.code()} {start_time.iso()} - {end_time.iso()}"
+                            )
+                            rs.addStream(
+                                net.code(),
+                                sta.code(),
+                                loc.code(),
+                                cha.code(),
+                                start_time,
+                                end_time,
+                                utils.isRestricted(cha),
+                                sta.archiveNetworkCode(),
+                            )
+
+        if forbidden:
+            for tracker in trackerList:
+                tracker.volume_status("fdsnws", "DENIED", 0, "")
+                tracker.request_status("END", "")
+
+            msg = "access denied"
+            return self.renderErrorPage(req, http.FORBIDDEN, msg, ro)
+
+        if forbidden is None:
+            for tracker in trackerList:
+                tracker.volume_status("fdsnws", "NODATA", 0, "")
+                tracker.request_status("END", "")
+
+            msg = "no metadata found"
+            return self.renderErrorPage(req, http.NO_CONTENT, msg, ro)
+
+        if auxStreamsFound:
+            msg = (
+                "the request contains at least one auxiliary stream which are "
+                "deprecated"
+            )
+            if maxSamples is not None:
+                msg += (
+                    " and whose samples are not included in the maximum sample per "
+                    "request limit"
+                )
+            logging.info(msg)
+
+        # Build output filename
+        fileName = (
+            Application.Instance()._fileNamePrefix.replace(
+                "%time", time.strftime("%Y-%m-%dT%H:%M:%S")
+            )
+            + ".mseed"
+        )
+
+        # Create producer for async IO
+        prod = _WaveformProducer(req, ro, rs, fileName, trackerList)
+        req.registerProducer(prod, True)
+        prod.resumeProducing()
+
+        # The request is handled by the deferred object
+        return server.NOT_DONE_YET
+
+
+# vim: ts=4 et

lib/python/seiscomp/fdsnws/fastsds.py

@@ -0,0 +1,216 @@
+################################################################################
+# Copyright (C) 2014-2017 by GFZ Potsdam
+#
+# Classes to access an SDS structure to be used by the Dataselect-WS
+#
+# Author:  Javier Quinteros
+# Email:   javier@gfz-potsdam.de
+################################################################################
+
+import datetime
+import os
+
+import seiscomp.logging
+import seiscomp.mseedlite
+
+
+class SDS:
+    def __init__(self, sdsRoot):
+        if isinstance(sdsRoot, list):
+            self.sdsRoot = sdsRoot
+
+        else:
+            self.sdsRoot = [sdsRoot]
+
+    def __getMSName(self, reqDate, net, sta, loc, cha):
+        for root in self.sdsRoot:
+            yield (
+                f"{root}/{reqDate.year}/{net}/{sta}/{cha}.D/{net}.{sta}.{loc}.{cha}.D."
+                f"{reqDate.year}.{reqDate.strftime('%j')}"
+            )
+
+    @staticmethod
+    def __time2recno(msFile, reclen, timeStart, recStart, timeEnd, recEnd, searchTime):
+        if searchTime <= timeStart:
+            msFile.seek(recStart * reclen)
+            rec = seiscomp.mseedlite.Record(msFile)
+            return (recStart, rec.end_time)
+
+        if searchTime >= timeEnd:
+            msFile.seek(recEnd * reclen)
+            rec = seiscomp.mseedlite.Record(msFile)
+            return (recEnd, rec.end_time)
+
+        t1 = timeStart
+        r1 = recStart
+        t2 = timeEnd
+        r2 = recEnd
+        rn = int(
+            r1
+            + (r2 - r1) * (searchTime - t1).total_seconds() / (t2 - t1).total_seconds()
+        )
+
+        rn = max(rn, recStart)
+        rn = min(rn, recEnd)
+
+        while True:
+            msFile.seek(rn * reclen)
+            rec = seiscomp.mseedlite.Record(msFile)
+
+            if rec.begin_time < searchTime:
+                r1 = rn
+                t1 = rec.begin_time
+
+                if t1 == t2:
+                    break
+
+                rn = int(
+                    r1
+                    + (r2 - r1)
+                    * (searchTime - t1).total_seconds()
+                    / (t2 - t1).total_seconds()
+                )
+
+                rn = max(rn, recStart)
+                rn = min(rn, recEnd)
+
+                if rn == r1:
+                    break
+
+            else:
+                r2 = rn
+                t2 = rec.begin_time
+
+                if t1 == t2:
+                    break
+
+                rn = int(
+                    r2
+                    - (r2 - r1)
+                    * (t2 - searchTime).total_seconds()
+                    / (t2 - t1).total_seconds()
+                )
+
+                rn = max(rn, recStart)
+                rn = min(rn, recEnd)
+
+                if rn == r2:
+                    break
+
+        return rn, rec.end_time
+
+    def __getWaveform(self, startt, endt, msFile, bufferSize):
+        if startt >= endt:
+            return
+
+        rec = seiscomp.mseedlite.Record(msFile)
+        reclen = rec.size
+        recStart = 0
+        timeStart = rec.begin_time
+
+        if rec.begin_time >= endt:
+            return
+
+        msFile.seek(-reclen, 2)
+        rec = seiscomp.mseedlite.Record(msFile)
+        recEnd = msFile.tell() // reclen - 1
+        timeEnd = rec.begin_time
+
+        if rec.end_time <= startt:
+            return
+
+        if timeStart >= timeEnd:
+            seiscomp.logging.error(
+                f"{msFile.name}: overlap detected (start={timeStart}, end={timeEnd})"
+            )
+            return
+
+        (lower, _) = self.__time2recno(
+            msFile, reclen, timeStart, recStart, timeEnd, recEnd, startt
+        )
+        (upper, _) = self.__time2recno(
+            msFile, reclen, startt, lower, timeEnd, recEnd, endt
+        )
+
+        if upper < lower:
+            seiscomp.logging.error(
+                f"{msFile.name}: overlap detected (lower={lower}, upper={upper})"
+            )
+            upper = lower
+
+        msFile.seek(lower * reclen)
+        remaining = (upper - lower + 1) * reclen
+        check = True
+
+        if bufferSize % reclen:
+            bufferSize += reclen - bufferSize % reclen
+
+        while remaining > 0:
+            size = min(remaining, bufferSize)
+            data = msFile.read(size)
+            remaining -= size
+            offset = 0
+
+            if not data:
+                return
+
+            if check:
+                while offset < len(data):
+                    rec = seiscomp.mseedlite.Record(data[offset : offset + reclen])
+
+                    if rec.begin_time >= endt:
+                        return
+
+                    if rec.end_time > startt:
+                        break
+
+                    offset += reclen
+
+                check = False
+
+            if offset < len(data):
+                yield data[offset:] if offset else data
+
+        while True:
+            data = msFile.read(reclen)
+
+            if not data:
+                return
+
+            rec = seiscomp.mseedlite.Record(data)
+
+            if rec.begin_time >= endt:
+                return
+
+            yield data
+
+    def __getDayRaw(self, day, startt, endt, net, sta, loc, cha, bufferSize):
+        # Take into account the case of empty location
+        if loc == "--":
+            loc = ""
+
+        for dataFile in self.__getMSName(day, net, sta, loc, cha):
+            if not os.path.exists(dataFile):
+                continue
+
+            try:
+                with open(dataFile, "rb") as msFile:
+                    for buf in self.__getWaveform(startt, endt, msFile, bufferSize):
+                        yield buf
+
+            except seiscomp.mseedlite.MSeedError as e:
+                seiscomp.logging.error(f"{dataFile}: {e}")
+
+    def getRawBytes(self, startt, endt, net, sta, loc, cha, bufferSize):
+        day = datetime.datetime(
+            startt.year, startt.month, startt.day
+        ) - datetime.timedelta(days=1)
+        endDay = datetime.datetime(endt.year, endt.month, endt.day)
+
+        while day <= endDay:
+            for buf in self.__getDayRaw(
+                day, startt, endt, net, sta, loc, cha, bufferSize
+            ):
+                yield buf
+
+            day += datetime.timedelta(days=1)

lib/python/seiscomp/fdsnws/http.py

@@ -0,0 +1,296 @@
+################################################################################
+# Copyright (C) 2013-2014 by gempa GmbH
+#
+# HTTP -- Utility methods which generate HTTP result strings
+#
+# Author:  Stephan Herrnkind
+# Email:   herrnkind@gempa.de
+################################################################################
+
+from twisted.web import http, resource, server, static, util
+
+import seiscomp.core
+import seiscomp.logging
+
+from .utils import accessLog, b_str, u_str, writeTSBin
+
+VERSION = "1.2.5"
+
+################################################################################
+
+
+class HTTP:
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def renderErrorPage(request, code, msg, version=VERSION, ro=None):
+        resp = b"""\
+Error %i: %s
+
+%s
+
+Usage details are available from %s
+
+Request:
+%s
+
+Request Submitted:
+%s
+
+Service Version:
+%s
+"""
+
+        noContent = code == http.NO_CONTENT
+
+        # rewrite response code if requested and no data was found
+        if noContent and ro is not None:
+            code = ro.noData
+
+        # set response code
+        request.setResponseCode(code)
+
+        # status code 204 requires no message body
+        if code == http.NO_CONTENT:
+            response = b""
+        else:
+            request.setHeader("Content-Type", "text/plain; charset=utf-8")
+
+            reference = b"%s/" % request.path.rpartition(b"/")[0]
+
+            codeStr = http.RESPONSES[code]
+            date = b_str(seiscomp.core.Time.GMT().toString("%FT%T.%f"))
+            response = resp % (
+                code,
+                codeStr,
+                b_str(msg),
+                reference,
+                request.uri,
+                date,
+                b_str(version),
+            )
+            if not noContent:
+                seiscomp.logging.warning(
+                    f"responding with error: {code} ({u_str(codeStr)})"
+                )
+
+        accessLog(request, ro, code, len(response), msg)
+        return response
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def renderNotFound(request, version=VERSION):
+        msg = "The requested resource does not exist on this server."
+        return HTTP.renderErrorPage(request, http.NOT_FOUND, msg, version)
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def renderNotModified(request, ro=None):
+        code = http.NOT_MODIFIED
+        request.setResponseCode(code)
+        request.responseHeaders.removeHeader("Content-Type")
+        accessLog(request, ro, code, 0, None)
+
+
+################################################################################
+class ServiceVersion(resource.Resource):
+    isLeaf = True
+
+    # ---------------------------------------------------------------------------
+    def __init__(self, version):
+        super().__init__()
+
+        self.version = version
+        self.type = "text/plain"
+
+    # ---------------------------------------------------------------------------
+    def render(self, request):
+        request.setHeader("Content-Type", "text/plain; charset=utf-8")
+        return b_str(self.version)
+
+
+################################################################################
+class WADLFilter(static.Data):
+    # ---------------------------------------------------------------------------
+    def __init__(self, path, paramNameFilterList):
+        data = ""
+        removeParam = False
+        with open(path, "r", encoding="utf-8") as fp:
+            for line in fp:
+                lineStripped = line.strip().replace(" ", "")
+                if removeParam:
+                    if "</param>" in lineStripped:
+                        removeParam = False
+                    continue
+
+                valid = True
+                if "<param" in lineStripped:
+                    for f in paramNameFilterList:
+                        if f'name="{f}"' in lineStripped:
+                            valid = False
+                            if lineStripped[-2:] != "/>":
+                                removeParam = True
+                            break
+
+                if valid:
+                    data += line
+
+        super().__init__(b_str(data), "application/xml; charset=utf-8")
+
+
+################################################################################
+class BaseResource(resource.Resource):
+    # ---------------------------------------------------------------------------
+    def __init__(self, version=VERSION):
+        super().__init__()
+
+        self.version = version
+
+    # ---------------------------------------------------------------------------
+    def renderErrorPage(self, request, code, msg, ro=None):
+        return HTTP.renderErrorPage(request, code, msg, self.version, ro)
+
+    # ---------------------------------------------------------------------------
+    def writeErrorPage(self, request, code, msg, ro=None):
+        data = self.renderErrorPage(request, code, msg, ro)
+        if data:
+            writeTSBin(request, data)
+
+    # ---------------------------------------------------------------------------
+    def returnNotModified(self, request, ro=None):
+        HTTP.renderNotModified(request, ro)
+
+    # ---------------------------------------------------------------------------
+    # Renders error page if the result set exceeds the configured maximum number
+    # objects
+    def checkObjects(self, request, objCount, maxObj):
+        if objCount <= maxObj:
+            return True
+
+        msg = (
+            "The result set of your request exceeds the configured maximum "
+            f"number of objects ({maxObj}). Refine your request parameters."
+        )
+        self.writeErrorPage(request, http.REQUEST_ENTITY_TOO_LARGE, msg)
+        return False
+
+
+################################################################################
+class NoResource(BaseResource):
+    isLeaf = True
+
+    # ---------------------------------------------------------------------------
+    def render(self, request):
+        return HTTP.renderNotFound(request, self.version)
+
+    # ---------------------------------------------------------------------------
+    def getChild(self, _path, _request):
+        return self
+
+
+################################################################################
+class ListingResource(BaseResource):
+    html = """<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="author" content="gempa GmbH">
+  <title>SeisComP FDSNWS Implementation</title>
+</head>
+<body>
+  <p><a href="../">Parent Directory</a></p>
+  <h1>SeisComP FDSNWS Web Service</h1>
+  <p>Index of %s</p>
+  <ul>
+%s
+  </ul>
+</body>"""
+
+    # ---------------------------------------------------------------------------
+    def render(self, request):
+        lis = ""
+        if request.path[-1:] != b"/":
+            return util.redirectTo(request.path + b"/", request)
+
+        for k, v in self.children.items():
+            if v.isLeaf:
+                continue
+            if hasattr(v, "hideInListing") and v.hideInListing:
+                continue
+            name = u_str(k)
+            lis += f'<li><a href="{name}/">{name}/</a></li>\n'
+
+        return b_str(ListingResource.html % (u_str(request.path), lis))
+
+    # ---------------------------------------------------------------------------
+    def getChild(self, path, _request):
+        if not path:
+            return self
+
+        return NoResource(self.version)
+
+
+################################################################################
+class DirectoryResource(static.File):
+    # ---------------------------------------------------------------------------
+    def __init__(self, fileName, version=VERSION):
+        super().__init__(fileName)
+
+        self.version = version
+        self.childNotFound = NoResource(self.version)
+
+    # ---------------------------------------------------------------------------
+    def render(self, request):
+        if request.path[-1:] != b"/":
+            return util.redirectTo(request.path + b"/", request)
+
+        return static.File.render(self, request)
+
+    # ---------------------------------------------------------------------------
+    def getChild(self, path, _request):
+        if not path:
+            return self
+
+        return NoResource(self.version)
+
+
+################################################################################
+class Site(server.Site):
+    def __init__(self, res, corsOrigins):
+        super().__init__(res)
+
+        self._corsOrigins = corsOrigins
+
+    # ---------------------------------------------------------------------------
+    def getResourceFor(self, request):
+        seiscomp.logging.debug(
+            f"request ({request.getClientIP()}): {u_str(request.uri)}"
+        )
+        request.setHeader("Server", f"SeisComP-FDSNWS/{VERSION}")
+        request.setHeader("Access-Control-Allow-Headers", "Authorization")
+        request.setHeader("Access-Control-Expose-Headers", "WWW-Authenticate")
+
+        self.setAllowOrigin(request)
+
+        return server.Site.getResourceFor(self, request)
+
+    # ---------------------------------------------------------------------------
+    def setAllowOrigin(self, req):
+        # no allowed origin: no response header
+        lenOrigins = len(self._corsOrigins)
+        if lenOrigins == 0:
+            return
+
+        # one origin: add header
+        if lenOrigins == 1:
+            req.setHeader("Access-Control-Allow-Origin", self._corsOrigins[0])
+            return
+
+        # more than one origin: check current origin against allowed origins
+        # and return the current origin on match.
+        origin = req.getHeader("Origin")
+        if origin in self._corsOrigins:
+            req.setHeader("Access-Control-Allow-Origin", origin)
+
+            # Set Vary header to let the browser know that the response depends
+            # on the request. Certain cache strategies should be disabled.
+            req.setHeader("Vary", "Origin")

lib/python/seiscomp/fdsnws/log.py

@@ -0,0 +1,101 @@
+################################################################################
+# Copyright (C) 2013-2014 gempa GmbH
+#
+# Thread-safe file logger
+#
+# Author:  Stephan Herrnkind
+# Email:   herrnkind@gempa.de
+################################################################################
+
+import os
+import sys
+import time
+import threading
+
+from queue import Queue
+
+# -------------------------------------------------------------------------------
+
+
+def _worker(log):
+    while True:
+        # pylint: disable=W0212
+        msg = log._queue.get()
+        log._write(str(msg))
+        log._queue.task_done()
+
+
+################################################################################
+class Log:
+    # ---------------------------------------------------------------------------
+    def __init__(self, filePath, archiveSize=7):
+        self._filePath = filePath
+        self._basePath = os.path.dirname(filePath)
+        self._fileName = os.path.basename(filePath)
+        self._archiveSize = archiveSize
+        self._queue = Queue()
+        self._lastLogTime = None
+        self._fd = None
+
+        self._archiveSize = max(self._archiveSize, 0)
+
+        # worker thread, responsible for writing messages to file
+        t = threading.Thread(target=_worker, args=(self,))
+        t.daemon = True
+        t.start()
+
+    # ---------------------------------------------------------------------------
+    def __del__(self):
+        # wait for worker thread to write all pending log messages
+        self._queue.join()
+
+        if self._fd is not None:
+            self._fd.close()
+
+    # ---------------------------------------------------------------------------
+    def log(self, msg):
+        self._queue.put(msg)
+
+    # ---------------------------------------------------------------------------
+    def _rotate(self):
+        self._fd.close()
+        self._fd = None
+
+        try:
+            pattern = f"{self._filePath}.%i"
+            for i in range(self._archiveSize, 1, -1):
+                src = pattern % (i - 1)
+                if os.path.isfile(src):
+                    os.rename(pattern % (i - 1), pattern % i)
+            os.rename(self._filePath, pattern % 1)
+        except Exception as e:
+            print(f"failed to rotate access log: {e}", file=sys.stderr)
+
+        self._fd = open(self._filePath, "w", encoding="utf-8")
+
+    # ---------------------------------------------------------------------------
+    def _write(self, msg):
+        try:
+            now = time.localtime()
+            if self._fd is None:
+                if self._basePath and not os.path.exists(self._basePath):
+                    os.makedirs(self._basePath)
+                self._fd = open(self._filePath, "a", encoding="utf-8")
+            elif (
+                self._archiveSize > 0
+                and self._lastLogTime is not None
+                and (
+                    self._lastLogTime.tm_yday != now.tm_yday
+                    or self._lastLogTime.tm_year != now.tm_year
+                )
+            ):
+                self._rotate()
+
+            print(msg, file=self._fd)
+            self._fd.flush()
+            self._lastLogTime = now
+        except Exception as e:
+            print(f"access log: {e}", file=sys.stderr)
+
+
+# vim: ts=4 et

lib/python/seiscomp/fdsnws/reqlog.py

@@ -0,0 +1,138 @@
+import os
+import datetime
+import json
+import hashlib
+import subprocess
+import logging
+import logging.handlers
+import threading
+
+
+from .utils import b_str
+
+mutex = threading.Lock()
+
+
+class MyFileHandler(logging.handlers.TimedRotatingFileHandler):
+    def __init__(self, filename):
+        super().__init__(filename, when="midnight", utc=True)
+
+    def rotate(self, source, dest):
+        super().rotate(source, dest)
+
+        if os.path.exists(dest):
+            subprocess.Popen(["bzip2", dest])
+
+
+class Tracker:
+    def __init__(self, logger, geoip, service, userName, userIP, clientID, userSalt):
+        self.__logger = logger
+        self.__userName = userName
+        self.__userSalt = userSalt
+        self.__logged = False
+
+        if userName:
+            userID = int(
+                hashlib.md5(b_str(userSalt + userName.lower())).hexdigest()[:8], 16
+            )
+        else:
+            userID = int(hashlib.md5(b_str(userSalt + userIP)).hexdigest()[:8], 16)
+
+        self.__data = {
+            "service": service,
+            "userID": userID,
+            "clientID": clientID,
+            "userEmail": None,
+            "auth": bool(userName),
+            "userLocation": {},
+            "created": f"{datetime.datetime.utcnow().isoformat()}Z",
+        }
+
+        if geoip:
+            self.__data["userLocation"]["country"] = geoip.country_code_by_addr(userIP)
+
+        if (
+            userName and userName.lower().endswith("@gfz-potsdam.de")
+        ) or userIP.startswith("139.17."):
+            self.__data["userLocation"]["institution"] = "GFZ"
+
+    # pylint: disable=W0613
+    def line_status(
+        self,
+        start_time,
+        end_time,
+        network,
+        station,
+        channel,
+        location,
+        restricted,
+        net_class,
+        shared,
+        constraints,
+        volume,
+        status,
+        size,
+        message,
+    ):
+        try:
+            trace = self.__data["trace"]
+
+        except KeyError:
+            trace = []
+            self.__data["trace"] = trace
+
+        trace.append(
+            {
+                "net": network,
+                "sta": station,
+                "loc": location,
+                "cha": channel,
+                "start": start_time.iso(),
+                "end": end_time.iso(),
+                "restricted": restricted,
+                "status": status,
+                "bytes": size,
+            }
+        )
+
+        if restricted and status == "OK":
+            self.__data["userEmail"] = self.__userName
+
+    # FDSNWS requests have one volume, so volume_status() is called once per request
+    def volume_status(self, volume, status, size, message):
+        self.__data["status"] = status
+        self.__data["bytes"] = size
+        self.__data["finished"] = f"{datetime.datetime.utcnow().isoformat()}Z"
+
+    def request_status(self, status, message):
+        with mutex:
+            if not self.__logged:
+                self.__logger.info(json.dumps(self.__data))
+                self.__logged = True
+
+
+class RequestLog:
+    def __init__(self, filename, userSalt):
+        self.__logger = logging.getLogger("seiscomp.fdsnws.reqlog")
+        self.__logger.addHandler(MyFileHandler(filename))
+        self.__logger.setLevel(logging.INFO)
+        self.__userSalt = userSalt
+
+        try:
+            import GeoIP
+
+            self.__geoip = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
+
+        except ImportError:
+            self.__geoip = None
+
+    def tracker(self, service, userName, userIP, clientID):
+        return Tracker(
+            self.__logger,
+            self.__geoip,
+            service,
+            userName,
+            userIP,
+            clientID,
+            self.__userSalt,
+        )

lib/python/seiscomp/fdsnws/reqtrack.py

@@ -0,0 +1,179 @@
+from twisted.internet import reactor
+import seiscomp.core
+import seiscomp.datamodel
+
+
+def callFromThread(f):
+    def wrap(*args, **kwargs):
+        reactor.callFromThread(f, *args, **kwargs)
+
+    return wrap
+
+
+def enableNotifier(f):
+    def wrap(*args, **kwargs):
+        saveState = seiscomp.datamodel.Notifier.IsEnabled()
+        seiscomp.datamodel.Notifier.SetEnabled(True)
+        f(*args, **kwargs)
+        seiscomp.datamodel.Notifier.SetEnabled(saveState)
+
+    return wrap
+
+
+class RequestTrackerDB(object):
+    def __init__(
+        self,
+        appName,
+        msgConn,
+        req_id,
+        req_type,
+        user,
+        header,
+        label,
+        user_ip,
+        client_ip,
+    ):
+        self.msgConn = msgConn
+        self.arclinkRequest = seiscomp.datamodel.ArclinkRequest.Create()
+        self.arclinkRequest.setCreated(seiscomp.core.Time.GMT())
+        self.arclinkRequest.setRequestID(req_id)
+        self.arclinkRequest.setUserID(str(user))
+        self.arclinkRequest.setClientID(appName)
+        if user_ip:
+            self.arclinkRequest.setUserIP(user_ip)
+        if client_ip:
+            self.arclinkRequest.setClientIP(client_ip)
+        self.arclinkRequest.setType(req_type)
+        self.arclinkRequest.setLabel(label)
+        self.arclinkRequest.setHeader(header)
+
+        self.averageTimeWindow = seiscomp.core.TimeSpan(0.0)
+        self.totalLineCount = 0
+        self.okLineCount = 0
+
+        self.requestLines = []
+        self.statusLines = []
+
+    def send(self):
+        msg = seiscomp.datamodel.Notifier.GetMessage(True)
+        if msg:
+            self.msgConn.send("LOGGING", msg)
+
+    def line_status(
+        self,
+        start_time,
+        end_time,
+        network,
+        station,
+        channel,
+        location,
+        restricted,
+        net_class,
+        shared,
+        constraints,
+        volume,
+        status,
+        size,
+        message,
+    ):
+        if network is None or network == "":
+            network = "."
+        if station is None or station == "":
+            station = "."
+        if channel is None or channel == "":
+            channel = "."
+        if location is None or location == "":
+            location = "."
+        if volume is None:
+            volume = "NODATA"
+        if size is None:
+            size = 0
+        if message is None:
+            message = ""
+
+        if isinstance(constraints, list):
+            constr = " ".join(constraints)
+        else:
+            constr = " ".join([f"{a}={b}" for (a, b) in constraints.items()])
+
+        arclinkRequestLine = seiscomp.datamodel.ArclinkRequestLine()
+        arclinkRequestLine.setStart(start_time)
+        arclinkRequestLine.setEnd(end_time)
+        arclinkRequestLine.setStreamID(
+            seiscomp.datamodel.WaveformStreamID(
+                network[:8], station[:8], location[:8], channel[:8], ""
+            )
+        )
+        arclinkRequestLine.setConstraints(constr)
+        if isinstance(restricted, bool):
+            arclinkRequestLine.setRestricted(restricted)
+        arclinkRequestLine.setNetClass(net_class)
+        if isinstance(shared, bool):
+            arclinkRequestLine.setShared(shared)
+        #
+        arclinkStatusLine = seiscomp.datamodel.ArclinkStatusLine()
+        arclinkStatusLine.setVolumeID(volume)
+        arclinkStatusLine.setStatus(status)
+        arclinkStatusLine.setSize(size)
+        arclinkStatusLine.setMessage(message)
+        #
+        arclinkRequestLine.setStatus(arclinkStatusLine)
+        self.requestLines.append(arclinkRequestLine)
+
+        self.averageTimeWindow += end_time - start_time
+        self.totalLineCount += 1
+        if status == "OK":
+            self.okLineCount += 1
+
+    def volume_status(self, volume, status, size, message):
+        if volume is None:
+            volume = "NODATA"
+        if size is None:
+            size = 0
+        if message is None:
+            message = ""
+
+        arclinkStatusLine = seiscomp.datamodel.ArclinkStatusLine()
+        arclinkStatusLine.setVolumeID(volume)
+        arclinkStatusLine.setStatus(status)
+        arclinkStatusLine.setSize(size)
+        arclinkStatusLine.setMessage(message)
+        self.statusLines.append(arclinkStatusLine)
+
+    @callFromThread
+    @enableNotifier
+    def request_status(self, status, message):
+        if message is None:
+            message = ""
+
+        self.arclinkRequest.setStatus(status)
+        self.arclinkRequest.setMessage(message)
+
+        ars = seiscomp.datamodel.ArclinkRequestSummary()
+        tw = self.averageTimeWindow.seconds()
+        if self.totalLineCount > 0:
+            # avarage request time window
+            tw = self.averageTimeWindow.seconds() // self.totalLineCount
+        if tw >= 2**31:
+            tw = -1  # prevent 32bit int overflow
+        ars.setAverageTimeWindow(tw)
+        ars.setTotalLineCount(self.totalLineCount)
+        ars.setOkLineCount(self.okLineCount)
+        self.arclinkRequest.setSummary(ars)
+
+        al = seiscomp.datamodel.ArclinkLog()
+        al.add(self.arclinkRequest)
+
+        for obj in self.requestLines:
+            self.arclinkRequest.add(obj)
+
+        for obj in self.statusLines:
+            self.arclinkRequest.add(obj)
+
+        self.send()
+
+    def __verseed_errors(self, volume):
+        pass
+
+    def verseed(self, volume, file):
+        pass

lib/python/seiscomp/fdsnws/request.py

@@ -0,0 +1,609 @@
+################################################################################
+# Copyright (C) 2013-2014 gempa GmbH
+#
+# RequestOptions -- HTTP GET request parameters
+#
+# Author:  Stephan Herrnkind
+# Email:   herrnkind@gempa.de
+################################################################################
+
+import fnmatch
+import math
+import re
+
+from twisted.web import http
+
+from seiscomp.core import Time
+import seiscomp.logging
+import seiscomp.math
+
+from .utils import u_str
+
+
+class RequestOptions:
+    # the match() method matched only patterns at the beginning of a string,
+    # since we have to ensure that no invalid character is present we use the
+    # search() method in combination with a negated pattern instead
+    FloatChars = re.compile(r"[^-0-9.]").search
+    ChannelChars = re.compile(r"[^A-Za-z0-9*?]").search
+    ChannelExtChars = re.compile(r"[^A-Za-z0-9*?+\-_]").search
+    BooleanTrueValues = ["1", "true", "t", "yes", "y"]
+    BooleanFalseValues = ["0", "false", "f", "no", "n"]
+    OutputFormats = []  # override in derived classes
+
+    PStart = ["starttime", "start"]
+    PEnd = ["endtime", "end"]
+    PStartBefore = ["startbefore"]
+    PStartAfter = ["startafter"]
+    PEndBefore = ["endbefore"]
+    PEndAfter = ["endafter"]
+    SimpleTimeParams = PStart + PEnd
+    WindowTimeParams = PStartBefore + PStartAfter + PEndBefore + PEndAfter
+    TimeParams = SimpleTimeParams + WindowTimeParams
+
+    PNet = ["network", "net"]
+    PSta = ["station", "sta"]
+    PLoc = ["location", "loc"]
+    PCha = ["channel", "cha"]
+    StreamParams = PNet + PSta + PLoc + PCha
+
+    PMinLat = ["minlatitude", "minlat"]
+    PMaxLat = ["maxlatitude", "maxlat"]
+    PMinLon = ["minlongitude", "minlon"]
+    PMaxLon = ["maxlongitude", "maxlon"]
+    PLat = ["latitude", "lat"]
+    PLon = ["longitude", "lon"]
+    PMinRadius = ["minradius"]
+    PMaxRadius = ["maxradius"]
+    GeoRectParams = PMinLat + PMaxLat + PMinLon + PMaxLon
+    GeoCircleParams = PLat + PLon + PMinRadius + PMaxRadius
+    GeoParams = GeoRectParams + GeoCircleParams
+
+    PFormat = ["format"]
+    PNoData = ["nodata"]
+    OutputParams = PFormat + PNoData
+
+    POSTParams = OutputParams
+    GETParams = StreamParams + SimpleTimeParams
+
+    # ---------------------------------------------------------------------------
+    class Channel:
+        def __init__(self):
+            self.net = None
+            self.sta = None
+            self.loc = None
+            self.cha = None
+
+        def matchNet(self, value):
+            return self.match(value, self.net)
+
+        def matchSta(self, value):
+            return self.match(value, self.sta)
+
+        def matchLoc(self, value):
+            return self.match(value, self.loc, True)
+
+        def matchCha(self, value):
+            return self.match(value, self.cha)
+
+        @staticmethod
+        def match(value, globList, testEmpty=False):
+            if not globList:
+                return True
+
+            for glob in globList:
+                if testEmpty and value == "" and glob == "--":
+                    return True
+                if fnmatch.fnmatchcase(value, glob):
+                    return True
+
+            return False
+
+    # ---------------------------------------------------------------------------
+    class Time:
+        def __init__(self):
+            self.simpleTime = True
+            self.start = None
+            self.end = None
+            # window time only
+            self.startBefore = None
+            self.startAfter = None
+            self.endBefore = None
+            self.endAfter = None
+
+        # used by FDSN Station and DataSelect
+        def match(self, start, end=None):
+            # simple time: limit to epochs intersecting with the specified time
+            # range
+            res = (self.start is None or end is None or end >= self.start) and (
+                self.end is None or start <= self.end
+            )
+
+            # window time: limit to epochs strictly starting or ending before or
+            # after a specified time value
+            if not self.simpleTime:
+                res = (
+                    res
+                    and (
+                        self.startBefore is None
+                        or (start is not None and start < self.startBefore)
+                    )
+                    and (
+                        self.startAfter is None
+                        or (start is not None and start > self.startAfter)
+                    )
+                    and (
+                        self.endBefore is None
+                        or (end is not None and end < self.endBefore)
+                    )
+                    and (self.endAfter is None or end is None or end > self.endAfter)
+                )
+
+            return res
+
+    # ---------------------------------------------------------------------------
+    class Geo:
+        # -----------------------------------------------------------------------
+        class BBox:
+            def __init__(self):
+                self.minLat = None
+                self.maxLat = None
+                self.minLon = None
+                self.maxLon = None
+
+            def dateLineCrossing(self):
+                return self.minLon and self.maxLon and self.minLon > self.maxLon
+
+        # -----------------------------------------------------------------------
+        class BCircle:
+            def __init__(self):
+                self.lat = None
+                self.lon = None
+                self.minRad = None
+                self.maxRad = None
+
+            # -------------------------------------------------------------------
+            # Calculates outer bounding box
+            def calculateBBox(self):
+                def rad(degree):
+                    return math.radians(degree)
+
+                def deg(radians):
+                    return math.degrees(radians)
+
+                b = RequestOptions.Geo.BBox()
+                if self.maxRad is None or self.maxRad >= 180:
+                    return b
+
+                b.minLat = self.lat - self.maxRad
+                b.maxLat = self.lat + self.maxRad
+                if b.minLat > -90 and b.maxLat < 90:
+                    dLon = deg(
+                        math.asin(math.sin(rad(self.maxRad) / math.cos(rad(self.lat))))
+                    )
+                    b.minLon = self.lon - dLon
+                    if b.minLon < -180:
+                        b.minLon += 360
+                    b.maxLon = self.lon + dLon
+                    if b.maxLon > 180:
+                        b.maxLon -= 360
+                else:
+                    # pole within distance: one latitude and no longitude
+                    # restrictions remains
+                    if b.minLat <= -90:
+                        b.minLat = None
+                    else:
+                        b.maxLat = None
+                    b.minLon = None
+                    b.maxLon = None
+
+                return b
+
+        # -----------------------------------------------------------------------
+        def __init__(self):
+            self.bBox = None
+            self.bCircle = None
+
+        # -----------------------------------------------------------------------
+        def match(self, lat, lon):
+            if self.bBox is not None:
+                b = self.bBox
+                if b.minLat is not None and lat < b.minLat:
+                    return False
+                if b.maxLat is not None and lat > b.maxLat:
+                    return False
+                # date line crossing if minLon > maxLon
+                if b.dateLineCrossing():
+                    return lon >= b.minLon or lon <= b.maxLon
+                if b.minLon is not None and lon < b.minLon:
+                    return False
+                if b.maxLon is not None and lon > b.maxLon:
+                    return False
+                return True
+
+            if self.bCircle:
+                c = self.bCircle
+                dist = seiscomp.math.delazi(c.lat, c.lon, lat, lon)
+                if c.minRad is not None and dist[0] < c.minRad:
+                    return False
+                if c.maxRad is not None and dist[0] > c.maxRad:
+                    return False
+                return True
+
+            return False
+
+    # ---------------------------------------------------------------------------
+    def __init__(self):
+        self.service = ""
+        self.accessTime = Time.GMT()
+        self.userName = None
+
+        self.time = None
+        self.channel = None
+        self.geo = None
+
+        self.noData = http.NO_CONTENT
+        self.format = None
+
+        self._args = {}
+        self.streams = []  # 1 entry for GET, multipl
+
+    # ---------------------------------------------------------------------------
+    def parseOutput(self):
+        # nodata
+        code = self.parseInt(self.PNoData)
+        if code is not None:
+            if code not in (http.NO_CONTENT, http.NOT_FOUND):
+                self.raiseValueError(self.PNoData[0])
+            self.noData = code
+
+        # format
+        key, value = self.getFirstValue(self.PFormat)
+        if value is None:
+            # no format specified: default to first in list if available
+            if len(self.OutputFormats) > 0:
+                self.format = self.OutputFormats[0]
+        else:
+            value = value.lower()
+            if value in self.OutputFormats:
+                self.format = value
+            else:
+                self.raiseValueError(key)
+
+    # ---------------------------------------------------------------------------
+    def parseChannel(self):
+        c = RequestOptions.Channel()
+
+        c.net = self.parseChannelChars(self.PNet, False, True)
+        c.sta = self.parseChannelChars(self.PSta)
+        c.loc = self.parseChannelChars(self.PLoc, True)
+        c.cha = self.parseChannelChars(self.PCha)
+
+        if c.net or c.sta or c.loc or c.cha:
+            self.channel = c
+
+    # ---------------------------------------------------------------------------
+    def parseTime(self, parseWindowTime=False):
+        t = RequestOptions.Time()
+
+        # start[time], end[time]
+        t.start = self.parseTimeStr(self.PStart)
+        t.end = self.parseTimeStr(self.PEnd)
+
+        simpleTime = t.start is not None or t.end is not None
+
+        # [start,end][before,after]
+        if parseWindowTime:
+            t.startBefore = self.parseTimeStr(self.PStartBefore)
+            t.startAfter = self.parseTimeStr(self.PStartAfter)
+            t.endBefore = self.parseTimeStr(self.PEndBefore)
+            t.endAfter = self.parseTimeStr(self.PEndAfter)
+
+            windowTime = (
+                t.startBefore is not None
+                or t.startAfter is not None
+                or t.endBefore is not None
+                or t.endAfter is not None
+            )
+            if simpleTime or windowTime:
+                self.time = t
+                self.time.simpleTime = not windowTime
+
+        elif simpleTime:
+            self.time = t
+            self.time.simpleTime = True
+
+    # ---------------------------------------------------------------------------
+    def parseGeo(self):
+        # bounding box (optional)
+        b = RequestOptions.Geo.BBox()
+        b.minLat = self.parseFloat(self.PMinLat, -90, 90)
+        b.maxLat = self.parseFloat(self.PMaxLat, -90, 90)
+        if b.minLat is not None and b.maxLat is not None and b.minLat > b.maxLat:
+            raise ValueError(f"{self.PMinLat[0]} exceeds {self.PMaxLat[0]}")
+
+        b.minLon = self.parseFloat(self.PMinLon, -180, 180)
+        b.maxLon = self.parseFloat(self.PMaxLon, -180, 180)
+        # maxLon < minLon -> date line crossing
+
+        hasBBoxParam = (
+            b.minLat is not None
+            or b.maxLat is not None
+            or b.minLon is not None
+            or b.maxLon is not None
+        )
+
+        # bounding circle (optional)
+        c = RequestOptions.Geo.BCircle()
+        c.lat = self.parseFloat(self.PLat, -90, 90)
+        c.lon = self.parseFloat(self.PLon, -180, 180)
+        c.minRad = self.parseFloat(self.PMinRadius, 0, 180)
+        c.maxRad = self.parseFloat(self.PMaxRadius, 0, 180)
+        if c.minRad is not None and c.maxRad is not None and c.minRad > c.maxRad:
+            raise ValueError(f"{self.PMinRadius[0]} exceeds {self.PMaxRadius[0]}")
+
+        hasBCircleRadParam = c.minRad is not None or c.maxRad is not None
+        hasBCircleParam = c.lat is not None or c.lon is not None or hasBCircleRadParam
+
+        # bounding box and bounding circle may not be combined
+        if hasBBoxParam and hasBCircleParam:
+            raise ValueError(
+                "bounding box and bounding circle parameters may not be combined"
+            )
+        if hasBBoxParam:
+            self.geo = RequestOptions.Geo()
+            self.geo.bBox = b
+        elif hasBCircleRadParam:
+            self.geo = RequestOptions.Geo()
+            if c.lat is None:
+                c.lat = 0.0
+            if c.lon is None:
+                c.lon = 0.0
+            self.geo.bCircle = c
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def _assertValueRange(key, v, minValue, maxValue):
+        if (minValue is not None and v < minValue) or (
+            maxValue is not None and v > maxValue
+        ):
+            minStr, maxStr = "-inf", "inf"
+            if minValue is not None:
+                minStr = str(minValue)
+            if maxValue is not None:
+                maxStr = str(maxValue)
+            raise ValueError(f"parameter not in domain [{minStr},{maxStr}]: {key}")
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def raiseValueError(key):
+        raise ValueError(f"invalid value in parameter: {key}")
+
+    # ---------------------------------------------------------------------------
+    def getFirstValue(self, keys):
+        for key in keys:
+            if key in self._args:
+                return key, self._args[key][0].strip()
+
+        return None, None
+
+    # ---------------------------------------------------------------------------
+    def getValues(self, keys):
+        v = []
+        for key in keys:
+            if key in self._args:
+                v += self._args[key]
+        return v
+
+    # ---------------------------------------------------------------------------
+    def getListValues(self, keys, lower=False):
+        values = set()
+        for key in keys:
+            if key not in self._args:
+                continue
+
+            for vList in self._args[key]:
+                for v in vList.split(","):
+                    if v is None:
+                        continue
+                    v = v.strip()
+                    if lower:
+                        v = v.lower()
+                    values.add(v)
+
+        return values
+
+    # ---------------------------------------------------------------------------
+    def parseInt(self, keys, minValue=None, maxValue=None):
+        key, value = self.getFirstValue(keys)
+
+        if value is None:
+            return None
+
+        try:
+            i = int(value)
+        except ValueError as e:
+            raise ValueError(f"invalid integer value in parameter: {key}") from e
+
+        self._assertValueRange(key, i, minValue, maxValue)
+        return i
+
+    # ---------------------------------------------------------------------------
+    def parseFloat(self, keys, minValue=None, maxValue=None):
+        key, value = self.getFirstValue(keys)
+
+        if value is None:
+            return None
+
+        if self.FloatChars(value):
+            raise ValueError(
+                f"invalid characters in float parameter: {key} (scientific notation "
+                "forbidden by spec)"
+            )
+
+        try:
+            f = float(value)
+        except ValueError as e:
+            raise ValueError(f"invalid float value in parameter: {key}") from e
+
+        self._assertValueRange(key, f, minValue, maxValue)
+        return f
+
+    # ---------------------------------------------------------------------------
+    def parseBool(self, keys):
+        key, value = self.getFirstValue(keys)
+
+        if value is None:
+            return None
+
+        value = value.lower()
+        if value in self.BooleanTrueValues:
+            return True
+        if value in self.BooleanFalseValues:
+            return False
+
+        raise ValueError(f"invalid boolean value in parameter: {key}")
+
+    # ---------------------------------------------------------------------------
+    def parseTimeStr(self, keys):
+        key, value = self.getFirstValue(keys)
+
+        if value is None:
+            return None
+
+        time = Time.FromString(value)
+        # use explicit test for None here since bool value for epoch date
+        # (1970-01-01) is False
+        if time is None:
+            raise ValueError(f"invalid date format in parameter: {key}")
+
+        return time
+
+    # ---------------------------------------------------------------------------
+    def parseChannelChars(self, keys, allowEmpty=False, useExtChars=False):
+        # channel parameters may be specified as a comma separated list and may
+        # be repeated several times
+        values = None
+        for vList in self.getValues(keys):
+            if values is None:
+                values = []
+            for v in vList.split(","):
+                v = v.strip()
+                if allowEmpty and (v == "--" or len(v) == 0):
+                    values.append("--")
+                    continue
+
+                if (useExtChars and self.ChannelExtChars(v)) or (
+                    not useExtChars and self.ChannelChars(v)
+                ):
+                    raise ValueError(f"invalid characters in parameter: {keys[0]}")
+                values.append(v)
+
+        return values
+
+    # ---------------------------------------------------------------------------
+    def parseGET(self, args):
+        # transform keys to lower case
+        if args is not None:
+            for k, v in args.items():
+                k = u_str(k.lower())
+                if k not in self.GETParams:
+                    raise ValueError(f"invalid param: {k}")
+
+                self._args[k] = [u_str(x) for x in v]
+
+    # ---------------------------------------------------------------------------
+    def parsePOST(self, content):
+        nLine = 0
+
+        for line in content:
+            nLine += 1
+            line = u_str(line.strip())
+
+            # ignore empty and comment lines
+            if len(line) == 0 or line[0] == "#":
+                continue
+
+            # collect parameter (non stream lines)
+            toks = line.split("=", 1)
+            if len(toks) > 1:
+                key = toks[0].strip().lower()
+
+                isPOSTParam = False
+                for p in self.POSTParams:
+                    if p == key:
+                        if key not in self._args:
+                            self._args[key] = []
+                        self._args[key].append(toks[1].strip())
+                        isPOSTParam = True
+                        break
+
+                if isPOSTParam:
+                    continue
+
+                # time parameters not allowed in POST header
+                for p in self.TimeParams:
+                    if p == key:
+                        raise ValueError(
+                            f"time parameter in line {nLine} not allowed in POST "
+                            "request"
+                        )
+
+                # stream parameters not allowed in POST header
+                for p in self.StreamParams:
+                    if p == key:
+                        raise ValueError(
+                            f"stream parameter in line {nLine} not allowed in POST "
+                            "request"
+                        )
+
+                raise ValueError(f"invalid parameter in line {nLine}")
+
+            # stream parameters
+            toks = line.split()
+            nToks = len(toks)
+            if nToks not in (5, 6):
+                raise ValueError("invalid number of stream components in line {nLine}")
+
+            ro = RequestOptions()
+
+            # net, sta, loc, cha
+            ro.channel = RequestOptions.Channel()
+            ro.channel.net = toks[0].split(",")
+            ro.channel.sta = toks[1].split(",")
+            ro.channel.loc = toks[2].split(",")
+            ro.channel.cha = toks[3].split(",")
+
+            msg = "invalid %s value in line %i"
+            for net in ro.channel.net:
+                if ro.ChannelChars(net):
+                    raise ValueError(msg % ("network", nLine))
+            for sta in ro.channel.sta:
+                if ro.ChannelChars(sta):
+                    raise ValueError(msg % ("station", nLine))
+            for loc in ro.channel.loc:
+                if loc != "--" and ro.ChannelChars(loc):
+                    raise ValueError(msg % ("location", nLine))
+            for cha in ro.channel.cha:
+                if ro.ChannelChars(cha):
+                    raise ValueError(msg % ("channel", nLine))
+
+            # start/end time
+            ro.time = RequestOptions.Time()
+            ro.time.start = Time.FromString(toks[4])
+            logEnd = "-"
+            if len(toks) > 5:
+                ro.time.end = Time.FromString(toks[5])
+                logEnd = ro.time.end.iso()
+
+            seiscomp.logging.debug(
+                f"ro: {ro.channel.net}.{ro.channel.sta}.{ro.channel.loc}."
+                f"{ro.channel.cha} {ro.time.start.iso()} {logEnd}"
+            )
+            self.streams.append(ro)
+
+        if not self.streams:
+            raise ValueError("at least one stream line is required")
+
+
+# vim: ts=4 et

lib/python/seiscomp/fdsnws/station.py

@@ -0,0 +1,936 @@
+################################################################################
+# Copyright (C) 2013-2014 gempa GmbH
+#
+# FDSNStation -- Implements the fdsnws-station Web service, see
+#   http://www.fdsn.org/webservices/
+#
+# Feature notes:
+#   - 'updatedafter' request parameter not implemented: The last modification
+#     time in SeisComP is tracked on the object level. If a child of an object
+#     is updated the update time is not propagated to all parents. In order to
+#     check if a station was updated all children must be evaluated recursively.
+#     This operation would be much to expensive.
+#   - additional request parameters:
+#     - formatted:       boolean, default: false
+#   - additional values of request parameters:
+#     - format
+#       - standard:      [xml, text]
+#       - additional:    [fdsnxml (=xml), stationxml, sc3ml]
+#       - default:       xml
+#
+# Author:  Stephan Herrnkind
+# Email:   herrnkind@gempa.de
+################################################################################
+
+from twisted.internet.threads import deferToThread
+from twisted.web import http, server
+
+import seiscomp.datamodel
+import seiscomp.logging
+from seiscomp.client import Application
+from seiscomp.core import Time
+from seiscomp.io import Exporter, ExportObjectList
+
+from .http import BaseResource
+from .request import RequestOptions
+from . import utils
+
+VERSION = "1.1.6"
+
+################################################################################
+
+
+class _StationRequestOptions(RequestOptions):
+    Exporters = {
+        "xml": "fdsnxml",
+        "fdsnxml": "fdsnxml",
+        "stationxml": "staxml",
+        "sc3ml": "trunk",
+    }
+    MinTime = Time(0, 1)
+
+    VText = ["text"]
+    # OutputFormats = list(Exporters) + VText
+    # Default format must be the first, list(Exporters) has random order
+    OutputFormats = ["xml", "fdsnxml", "stationxml", "sc3ml"] + VText
+
+    PLevel = ["level"]
+    PIncludeRestricted = ["includerestricted"]
+    PIncludeAvailability = ["includeavailability"]
+    PUpdateAfter = ["updateafter"]
+    PMatchTimeSeries = ["matchtimeseries"]
+
+    # non standard parameters
+    PFormatted = ["formatted"]
+
+    POSTParams = (
+        RequestOptions.POSTParams
+        + RequestOptions.GeoParams
+        + PLevel
+        + PIncludeRestricted
+        + PIncludeAvailability
+        + PUpdateAfter
+        + PMatchTimeSeries
+        + PFormatted
+    )
+    GETParams = RequestOptions.GETParams + RequestOptions.WindowTimeParams + POSTParams
+
+    # ---------------------------------------------------------------------------
+    def __init__(self):
+        super().__init__()
+
+        self.service = "fdsnws-station"
+
+        self.includeSta = True
+        self.includeCha = False
+        self.includeRes = False
+
+        self.restricted = None
+        self.availability = None
+        self.updatedAfter = None
+        self.matchTimeSeries = None
+
+        # non standard parameters
+        self.formatted = None
+
+    # ---------------------------------------------------------------------------
+    def parse(self):
+        self.parseTime(True)
+        self.parseChannel()
+        self.parseGeo()
+        self.parseOutput()
+
+        # level: [network, station, channel, response]
+        key, value = self.getFirstValue(self.PLevel)
+        if value is not None:
+            value = value.lower()
+            if value in ("network", "net"):
+                self.includeSta = False
+            elif value in ("channel", "cha", "chan"):
+                self.includeCha = True
+            elif value in ("response", "res", "resp"):
+                self.includeCha = True
+                self.includeRes = True
+            elif value not in ("station", "sta"):
+                self.raiseValueError(key)
+
+        # includeRestricted (optional)
+        self.restricted = self.parseBool(self.PIncludeRestricted)
+
+        # includeAvailability (optionalsc3ml)
+        self.availability = self.parseBool(self.PIncludeAvailability)
+
+        # updatedAfter (optional), currently not supported
+        self.updatedAfter = self.parseTimeStr(self.PUpdateAfter)
+
+        # includeAvailability (optional)
+        self.matchTimeSeries = self.parseBool(self.PMatchTimeSeries)
+
+        # format XML
+        self.formatted = self.parseBool(self.PFormatted)
+
+    # ---------------------------------------------------------------------------
+    def networkIter(self, inv, matchTime=False):
+        for i in range(inv.networkCount()):
+            net = inv.network(i)
+
+            for ro in self.streams:
+                # network code
+                if ro.channel and not ro.channel.matchNet(net.code()):
+                    continue
+
+                # start and end time
+                if matchTime and ro.time:
+                    try:
+                        end = net.end()
+                    except ValueError:
+                        end = None
+                    if not ro.time.match(net.start(), end):
+                        continue
+
+                yield net
+                break
+
+    # ---------------------------------------------------------------------------
+    def stationIter(self, net, matchTime=False):
+        for i in range(net.stationCount()):
+            sta = net.station(i)
+
+            # geographic location
+            if self.geo:
+                try:
+                    lat = sta.latitude()
+                    lon = sta.longitude()
+                except ValueError:
+                    continue
+                if not self.geo.match(lat, lon):
+                    continue
+
+            for ro in self.streams:
+                # station code
+                if ro.channel and (
+                    not ro.channel.matchSta(sta.code())
+                    or not ro.channel.matchNet(net.code())
+                ):
+                    continue
+
+                # start and end time
+                if matchTime and ro.time:
+                    try:
+                        end = sta.end()
+                    except ValueError:
+                        end = None
+                    if not ro.time.match(sta.start(), end):
+                        continue
+
+                yield sta
+                break
+
+    # ---------------------------------------------------------------------------
+    def locationIter(self, net, sta, matchTime=False):
+        for i in range(sta.sensorLocationCount()):
+            loc = sta.sensorLocation(i)
+
+            for ro in self.streams:
+                # location code
+                if ro.channel and (
+                    not ro.channel.matchLoc(loc.code())
+                    or not ro.channel.matchSta(sta.code())
+                    or not ro.channel.matchNet(net.code())
+                ):
+                    continue
+
+                # start and end time
+                if matchTime and ro.time:
+                    try:
+                        end = loc.end()
+                    except ValueError:
+                        end = None
+                    if not ro.time.match(loc.start(), end):
+                        continue
+
+                yield loc
+                break
+
+    # ---------------------------------------------------------------------------
+    def streamIter(self, net, sta, loc, matchTime, dac):
+        for i in range(loc.streamCount()):
+            stream = loc.stream(i)
+
+            for ro in self.streams:
+                # stream code
+                if ro.channel and (
+                    not ro.channel.matchCha(stream.code())
+                    or not ro.channel.matchLoc(loc.code())
+                    or not ro.channel.matchSta(sta.code())
+                    or not ro.channel.matchNet(net.code())
+                ):
+                    continue
+
+                # start and end time
+                if matchTime and ro.time:
+                    try:
+                        end = stream.end()
+                    except ValueError:
+                        end = None
+                    if not ro.time.match(stream.start(), end):
+                        continue
+
+                # match data availability extent
+                if dac is not None and self.matchTimeSeries:
+                    extent = dac.extent(
+                        net.code(), sta.code(), loc.code(), stream.code()
+                    )
+                    if extent is None or (
+                        ro.time and not ro.time.match(extent.start(), extent.end())
+                    ):
+                        continue
+
+                yield stream
+                break
+
+
+################################################################################
+class FDSNStation(BaseResource):
+    isLeaf = True
+
+    # ---------------------------------------------------------------------------
+    def __init__(
+        self,
+        inv,
+        restricted,
+        maxObj,
+        daEnabled,
+        conditionalRequestsEnabled,
+        timeInventoryLoaded,
+    ):
+        super().__init__(VERSION)
+
+        self._inv = inv
+        self._allowRestricted = restricted
+        self._maxObj = maxObj
+        self._daEnabled = daEnabled
+        self._conditionalRequestsEnabled = conditionalRequestsEnabled
+        self._timeInventoryLoaded = timeInventoryLoaded.seconds()
+
+        # additional object count dependent on detail level
+        self._resLevelCount = (
+            inv.responsePAZCount()
+            + inv.responseFIRCount()
+            + inv.responsePolynomialCount()
+            + inv.responseIIRCount()
+            + inv.responseFAPCount()
+        )
+        for i in range(inv.dataloggerCount()):
+            self._resLevelCount += inv.datalogger(i).decimationCount()
+
+    # ---------------------------------------------------------------------------
+    def render_OPTIONS(self, req):
+        req.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+        req.setHeader(
+            "Access-Control-Allow-Headers",
+            "Accept, Content-Type, X-Requested-With, Origin",
+        )
+        req.setHeader("Content-Type", "text/plain; charset=utf-8")
+        return ""
+
+    # ---------------------------------------------------------------------------
+    def render_GET(self, req):
+        # Parse and validate GET parameters
+        ro = _StationRequestOptions()
+        try:
+            ro.parseGET(req.args)
+            ro.parse()
+            # the GET operation supports exactly one stream filter
+            ro.streams.append(ro)
+        except ValueError as e:
+            seiscomp.logging.warning(str(e))
+            return self.renderErrorPage(req, http.BAD_REQUEST, str(e), ro)
+
+        return self._prepareRequest(req, ro)
+
+    # ---------------------------------------------------------------------------
+    def render_POST(self, req):
+        # Parse and validate POST parameters
+        ro = _StationRequestOptions()
+        try:
+            ro.parsePOST(req.content)
+            ro.parse()
+        except ValueError as e:
+            seiscomp.logging.warning(str(e))
+            return self.renderErrorPage(req, http.BAD_REQUEST, str(e), ro)
+
+        return self._prepareRequest(req, ro)
+
+    # ---------------------------------------------------------------------------
+    def _prepareRequest(self, req, ro):
+        if ro.availability and not self._daEnabled:
+            msg = "including of availability information not supported"
+            return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+
+        if ro.updatedAfter:
+            msg = "filtering based on update time not supported"
+            return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+
+        if ro.matchTimeSeries and not self._daEnabled:
+            msg = "filtering based on available time series not supported"
+            return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+
+        # load data availability if requested
+        dac = None
+        if ro.availability or ro.matchTimeSeries:
+            dac = Application.Instance().getDACache()
+            if dac is None or len(dac.extents()) == 0:
+                msg = "no data availabiltiy extent information found"
+                return self.renderErrorPage(req, http.NO_CONTENT, msg, ro)
+
+        # Exporter, 'None' is used for text output
+        if ro.format in ro.VText:
+            if ro.includeRes:
+                msg = "response level output not available in text format"
+                return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+            req.setHeader("Content-Type", "text/plain; charset=utf-8")
+            d = deferToThread(self._processRequestText, req, ro, dac)
+        else:
+            exp = Exporter.Create(ro.Exporters[ro.format])
+            if exp is None:
+                msg = (
+                    f"output format '{ro.format}' no available, export module "
+                    f"'{ro.Exporters[ro.format]}' could not be loaded."
+                )
+                return self.renderErrorPage(req, http.BAD_REQUEST, msg, ro)
+
+            req.setHeader("Content-Type", "application/xml; charset=utf-8")
+            exp.setFormattedOutput(bool(ro.formatted))
+            d = deferToThread(self._processRequestExp, req, ro, exp, dac)
+
+        req.notifyFinish().addErrback(utils.onCancel, d)
+        d.addBoth(utils.onFinish, req)
+
+        # The request is handled by the deferred object
+        return server.NOT_DONE_YET
+
+    # ---------------------------------------------------------------------------
+    def _processRequestExp(self, req, ro, exp, dac):
+        if req._disconnected:  # pylint: disable=W0212
+            return False
+
+        staCount, locCount, chaCount, extCount, objCount = 0, 0, 0, 0, 0
+
+        seiscomp.datamodel.PublicObject.SetRegistrationEnabled(False)
+        newInv = seiscomp.datamodel.Inventory()
+        dataloggers, sensors, extents = set(), set(), {}
+
+        skipRestricted = not self._allowRestricted or (
+            ro.restricted is not None and not ro.restricted
+        )
+        levelNet = not ro.includeSta
+        levelSta = ro.includeSta and not ro.includeCha
+
+        isConditionalRequest = self._isConditionalRequest(req)
+
+        # iterate over inventory networks
+        for net in ro.networkIter(self._inv, levelNet):
+            if req._disconnected:  # pylint: disable=W0212
+                return False
+            if skipRestricted and utils.isRestricted(net):
+                continue
+            newNet = seiscomp.datamodel.Network(net)
+
+            # Copy comments
+            for i in range(net.commentCount()):
+                newNet.add(seiscomp.datamodel.Comment(net.comment(i)))
+
+            # iterate over inventory stations of current network
+            for sta in ro.stationIter(net, levelSta):
+                if req._disconnected:  # pylint: disable=W0212
+                    return False
+                if skipRestricted and utils.isRestricted(sta):
+                    continue
+                if not self.checkObjects(req, objCount, self._maxObj):
+                    return False
+
+                if ro.includeCha:
+                    numCha, numLoc, d, s, e = self._processStation(
+                        newNet, net, sta, ro, dac, skipRestricted, isConditionalRequest
+                    )
+                    if numCha > 0:
+                        if isConditionalRequest:
+                            self.returnNotModified(req, ro)
+                            return True
+                        locCount += numLoc
+                        chaCount += numCha
+                        extCount += len(e)
+                        objCount += numLoc + numCha + extCount
+                        if not self.checkObjects(req, objCount, self._maxObj):
+                            return False
+                        dataloggers |= d
+                        sensors |= s
+                        for k, v in e.items():
+                            if k not in extents:
+                                extents[k] = v
+                elif self._matchStation(net, sta, ro, dac):
+                    if isConditionalRequest:
+                        self.returnNotModified(req, ro)
+                        return True
+                    if ro.includeSta:
+                        newSta = seiscomp.datamodel.Station(sta)
+                        # Copy comments
+                        for i in range(sta.commentCount()):
+                            newSta.add(seiscomp.datamodel.Comment(sta.comment(i)))
+                        newNet.add(newSta)
+                    else:
+                        # no station output requested: one matching station
+                        # is sufficient to include the network
+                        newInv.add(newNet)
+                        objCount += 1
+                        break
+
+            if newNet.stationCount() > 0:
+                newInv.add(newNet)
+                staCount += newNet.stationCount()
+                objCount += staCount + 1
+
+        # Return 204 if no matching inventory was found
+        if newInv.networkCount() == 0:
+            msg = "no matching inventory found"
+            self.writeErrorPage(req, http.NO_CONTENT, msg, ro)
+            return True
+
+        if self._conditionalRequestsEnabled:
+            req.setHeader(
+                "Last-Modified", http.datetimeToString(self._timeInventoryLoaded)
+            )
+
+        # Copy references (dataloggers, responses, sensors)
+        decCount, resCount = 0, 0
+        if ro.includeCha:
+            decCount = self._copyReferences(
+                newInv, req, objCount, self._inv, ro, dataloggers, sensors, self._maxObj
+            )
+            if decCount is None:
+                return False
+
+            resCount = (
+                newInv.responsePAZCount()
+                + newInv.responseFIRCount()
+                + newInv.responsePolynomialCount()
+                + newInv.responseFAPCount()
+                + newInv.responseIIRCount()
+            )
+            objCount += (
+                resCount + decCount + newInv.dataloggerCount() + newInv.sensorCount()
+            )
+
+        # Copy data extents
+        objOut = newInv
+        if len(extents) > 0:
+            objCount += 1
+            da = seiscomp.datamodel.DataAvailability()
+            for k, v in extents.items():
+                objCount += 1
+                da.add(seiscomp.datamodel.DataExtent(v))
+            objOut = ExportObjectList()
+            objOut.append(newInv)
+            objOut.append(da)
+
+        sink = utils.Sink(req)
+        if not exp.write(sink, objOut):
+            return False
+
+        seiscomp.logging.debug(
+            f"{ro.service}: returned {newInv.networkCount()}Net, {staCount}Sta, "
+            f"{locCount}Loc, {chaCount}Cha, {newInv.dataloggerCount()}DL, "
+            f"{decCount}Dec, {newInv.sensorCount()}Sen, {resCount}Res, {extCount}DAExt "
+            f"(total objects/bytes: {objCount}/{sink.written})"
+        )
+        utils.accessLog(req, ro, http.OK, sink.written, None)
+        return True
+
+    # ---------------------------------------------------------------------------
+    @staticmethod
+    def _formatEpoch(obj):
+        df = "%FT%T"
+        dfMS = "%FT%T.%f"
+
+        if obj.start().microseconds() > 0:
+            start = obj.start().toString(dfMS)
+        else:
+            start = obj.start().toString(df)
+
+        try:
+            if obj.end().microseconds() > 0:
+                end = obj.end().toString(dfMS)
+            else:
+                end = obj.end().toString(df)
+        except ValueError:
+            end = ""
+
+        return start, end
+
+    # ---------------------------------------------------------------------------
+    def _processRequestText(self, req, ro, dac):
+        if req._disconnected:  # pylint: disable=W0212
+            return False
+
+        skipRestricted = not self._allowRestricted or (
+            ro.restricted is not None and not ro.restricted
+        )
+        isConditionalRequest = self._isConditionalRequest(req)
+
+        data = ""
+        lines = []
+
+        # level = network
+        if not ro.includeSta:
+            data = "#Network|Description|StartTime|EndTime|TotalStations\n"
+
+            # iterate over inventory networks
+            for net in ro.networkIter(self._inv, True):
+                if req._disconnected:  # pylint: disable=W0212
+                    return False
+                if skipRestricted and utils.isRestricted(net):
+                    continue
+
+                # at least one matching station is required
+                stationFound = False
+                for sta in ro.stationIter(net, False):
+                    if req._disconnected:  # pylint: disable=W0212
+                        return False
+                    if self._matchStation(net, sta, ro, dac) and not (
+                        skipRestricted and utils.isRestricted(sta)
+                    ):
+                        stationFound = True
+                        break
+                if not stationFound:
+                    continue
+                if isConditionalRequest:
+                    self.returnNotModified(req, ro)
+                    return True
+
+                start, end = self._formatEpoch(net)
+                lines.append(
+                    (
+                        f"{net.code()} {start}",
+                        f"{net.code()}|{net.description()}|{start}|{end}|"
+                        f"{net.stationCount()}\n",
+                    )
+                )
+
+        # level = station
+        elif not ro.includeCha:
+            data = (
+                "#Network|Station|Latitude|Longitude|Elevation|"
+                "SiteName|StartTime|EndTime\n"
+            )
+
+            # iterate over inventory networks
+            for net in ro.networkIter(self._inv, False):
+                if req._disconnected:  # pylint: disable=W0212
+                    return False
+                if skipRestricted and utils.isRestricted(net):
+                    continue
+                # iterate over inventory stations
+                for sta in ro.stationIter(net, True):
+                    if req._disconnected:  # pylint: disable=W0212
+                        return False
+                    if not self._matchStation(net, sta, ro, dac) or (
+                        skipRestricted and utils.isRestricted(sta)
+                    ):
+                        continue
+                    if isConditionalRequest:
+                        self.returnNotModified(req, ro)
+                        return True
+
+                    try:
+                        lat = str(sta.latitude())
+                    except ValueError:
+                        lat = ""
+                    try:
+                        lon = str(sta.longitude())
+                    except ValueError:
+                        lon = ""
+                    try:
+                        elev = str(sta.elevation())
+                    except ValueError:
+                        elev = ""
+                    try:
+                        desc = sta.description()
+                    except ValueError:
+                        desc = ""
+
+                    start, end = self._formatEpoch(sta)
+                    lines.append(
+                        (
+                            f"{net.code()}.{sta.code()} {start}",
+                            f"{net.code()}|{sta.code()}|{lat}|{lon}|{elev}|{desc}|"
+                            f"{start}|{end}\n",
+                        )
+                    )
+
+        # level = channel (resonse level not supported in text format)
+        else:
+            data = (
+                "#Network|Station|Location|Channel|Latitude|Longitude|"
+                "Elevation|Depth|Azimuth|Dip|SensorDescription|Scale|"
+                "ScaleFreq|ScaleUnits|SampleRate|StartTime|EndTime\n"
+            )
+
+            # iterate over inventory networks
+            for net in ro.networkIter(self._inv, False):
+                if req._disconnected:  # pylint: disable=W0212
+                    return False
+                if skipRestricted and utils.isRestricted(net):
+                    continue
+                # iterate over inventory stations, locations, streams
+                for sta in ro.stationIter(net, False):
+                    if req._disconnected:  # pylint: disable=W0212
+                        return False
+                    if skipRestricted and utils.isRestricted(sta):
+                        continue
+                    for loc in ro.locationIter(net, sta, True):
+                        for stream in ro.streamIter(net, sta, loc, True, dac):
+                            if skipRestricted and utils.isRestricted(stream):
+                                continue
+                            if isConditionalRequest:
+                                self.returnNotModified(req, ro)
+                                return True
+
+                            try:
+                                lat = str(loc.latitude())
+                            except ValueError:
+                                lat = ""
+                            try:
+                                lon = str(loc.longitude())
+                            except ValueError:
+                                lon = ""
+                            try:
+                                elev = str(loc.elevation())
+                            except ValueError:
+                                elev = ""
+                            try:
+                                depth = str(stream.depth())
+                            except ValueError:
+                                depth = ""
+                            try:
+                                azi = str(stream.azimuth())
+                            except ValueError:
+                                azi = ""
+                            try:
+                                dip = str(stream.dip())
+                            except ValueError:
+                                dip = ""
+
+                            desc = ""
+                            try:
+                                sensor = self._inv.findSensor(stream.sensor())
+                                if sensor is not None:
+                                    desc = sensor.description()
+                            except ValueError:
+                                pass
+
+                            try:
+                                scale = str(stream.gain())
+                            except ValueError:
+                                scale = ""
+                            try:
+                                scaleFreq = str(stream.gainFrequency())
+                            except ValueError:
+                                scaleFreq = ""
+                            try:
+                                scaleUnit = str(stream.gainUnit())
+                            except ValueError:
+                                scaleUnit = ""
+                            try:
+                                sr = str(
+                                    stream.sampleRateNumerator()
+                                    / stream.sampleRateDenominator()
+                                )
+                            except (ValueError, ZeroDivisionError):
+                                sr = ""
+
+                            start, end = self._formatEpoch(stream)
+                            lines.append(
+                                (
+                                    f"{net.code()}.{sta.code()}.{loc.code()}."
+                                    f"{stream.code()} {start}",
+                                    f"{net.code()}|{sta.code()}|{loc.code()}|"
+                                    f"{stream.code()}|{lat}|{lon}|{elev}|{depth}|{azi}|"
+                                    f"{dip}|{desc}|{scale}|{scaleFreq}|{scaleUnit}|"
+                                    f"{sr}|{start}|{end}\n",
+                                )
+                            )
+
+        # sort lines and append to final data string
+        lines.sort(key=lambda line: line[0])
+        for line in lines:
+            data += line[1]
+
+        # Return 204 if no matching inventory was found
+        if len(lines) == 0:
+            msg = "no matching inventory found"
+            self.writeErrorPage(req, http.NO_CONTENT, msg, ro)
+            return False
+
+        if self._conditionalRequestsEnabled:
+            req.setHeader(
+                "Last-Modified", http.datetimeToString(self._timeInventoryLoaded)
+            )
+
+        dataBin = utils.b_str(data)
+        utils.writeTSBin(req, dataBin)
+        seiscomp.logging.debug(
+            f"{ro.service}: returned {len(lines)} lines (total bytes: {len(dataBin)})"
+        )
+        utils.accessLog(req, ro, http.OK, len(dataBin), None)
+        return True
+
+    # ---------------------------------------------------------------------------
+    def _isConditionalRequest(self, req):
+        # support for time based conditional requests
+        if not self._conditionalRequestsEnabled:
+            return False
+        if req.method not in (b"GET", b"HEAD"):
+            return False
+        if req.getHeader("If-None-Match") is not None:
+            return False
+
+        modifiedSince = req.getHeader("If-Modified-Since")
+        if not modifiedSince:
+            return False
+
+        modifiedSince = utils.stringToDatetime(modifiedSince)
+        return modifiedSince and self._timeInventoryLoaded <= modifiedSince
+
+    # ---------------------------------------------------------------------------
+    # Checks if at least one location and channel combination matches the
+    # request options
+    @staticmethod
+    def _matchStation(net, sta, ro, dac):
+        # No filter: return true immediately
+        if dac is None and (
+            not ro.channel or (not ro.channel.loc and not ro.channel.cha)
+        ):
+            return True
+
+        for loc in ro.locationIter(net, sta, False):
+            if dac is None and not ro.channel.cha and not ro.time:
+                return True
+
+            for _ in ro.streamIter(net, sta, loc, False, dac):
+                return True
+
+        return False
+
+    # ---------------------------------------------------------------------------
+    # Adds a deep copy of the specified station to the new network if the
+    # location and channel combination matches the request options (if any)
+    @staticmethod
+    def _processStation(
+        newNet, net, sta, ro, dac, skipRestricted, isConditionalRequest
+    ):
+        chaCount = 0
+        dataloggers, sensors, extents = set(), set(), {}
+        newSta = seiscomp.datamodel.Station(sta)
+        includeAvailability = dac is not None and ro.availability
+
+        # Copy comments
+        for i in range(sta.commentCount()):
+            newSta.add(seiscomp.datamodel.Comment(sta.comment(i)))
+
+        for loc in ro.locationIter(net, sta, True):
+            newLoc = seiscomp.datamodel.SensorLocation(loc)
+            # Copy comments
+            for i in range(loc.commentCount()):
+                newLoc.add(seiscomp.datamodel.Comment(loc.comment(i)))
+
+            for stream in ro.streamIter(net, sta, loc, True, dac):
+                if skipRestricted and utils.isRestricted(stream):
+                    continue
+                if isConditionalRequest:
+                    return 1, 1, [], [], []
+                newCha = seiscomp.datamodel.Stream(stream)
+                # Copy comments
+                for i in range(stream.commentCount()):
+                    newCha.add(seiscomp.datamodel.Comment(stream.comment(i)))
+                newLoc.add(newCha)
+                dataloggers.add(stream.datalogger())
+                sensors.add(stream.sensor())
+                if includeAvailability:
+                    ext = dac.extent(net.code(), sta.code(), loc.code(), stream.code())
+                    if ext is not None and ext.publicID() not in extents:
+                        extents[ext.publicID()] = ext
+
+            if newLoc.streamCount() > 0:
+                newSta.add(newLoc)
+                chaCount += newLoc.streamCount()
+
+        if newSta.sensorLocationCount() > 0:
+            newNet.add(newSta)
+            return chaCount, newSta.sensorLocationCount(), dataloggers, sensors, extents
+
+        return 0, 0, [], [], []
+
+    # ---------------------------------------------------------------------------
+    # Copy references (data loggers, sensors, responses) depended on request
+    # options
+    def _copyReferences(
+        self, newInv, req, objCount, inv, ro, dataloggers, sensors, maxObj
+    ):
+        responses = set()
+        decCount = 0
+
+        # datalogger
+        for i in range(inv.dataloggerCount()):
+            if req._disconnected:  # pylint: disable=W0212
+                return None
+            logger = inv.datalogger(i)
+            if logger.publicID() not in dataloggers:
+                continue
+            newLogger = seiscomp.datamodel.Datalogger(logger)
+            newInv.add(newLogger)
+            # decimations are only needed for responses
+            if ro.includeRes:
+                for j in range(logger.decimationCount()):
+                    decimation = logger.decimation(j)
+                    newLogger.add(seiscomp.datamodel.Decimation(decimation))
+
+                    # collect response ids
+                    filterStr = ""
+                    try:
+                        filterStr = f"{decimation.analogueFilterChain().content()} "
+                    except ValueError:
+                        pass
+                    try:
+                        filterStr += decimation.digitalFilterChain().content()
+                    except ValueError:
+                        pass
+                    for resp in filterStr.split():
+                        responses.add(resp)
+                decCount += newLogger.decimationCount()
+
+        objCount += newInv.dataloggerCount() + decCount
+        resCount = len(responses)
+        if not self.checkObjects(req, objCount + resCount, maxObj):
+            return None
+
+        # sensor
+        for i in range(inv.sensorCount()):
+            if req._disconnected:  # pylint: disable=W0212
+                return None
+            sensor = inv.sensor(i)
+            if sensor.publicID() not in sensors:
+                continue
+            newSensor = seiscomp.datamodel.Sensor(sensor)
+            newInv.add(newSensor)
+            resp = newSensor.response()
+            if resp:
+                if ro.includeRes:
+                    responses.add(resp)
+                else:
+                    # no responses: remove response reference to avoid missing
+                    # response warning of exporter
+                    newSensor.setResponse("")
+
+        objCount += newInv.sensorCount()
+        resCount = len(responses)
+        if not self.checkObjects(req, objCount + resCount, maxObj):
+            return None
+
+        # responses
+        if ro.includeRes:
+            if req._disconnected:  # pylint: disable=W0212
+                return None
+            for i in range(inv.responsePAZCount()):
+                resp = inv.responsePAZ(i)
+                if resp.publicID() in responses:
+                    newInv.add(seiscomp.datamodel.ResponsePAZ(resp))
+            if req._disconnected:  # pylint: disable=W0212
+                return None
+            for i in range(inv.responseFIRCount()):
+                resp = inv.responseFIR(i)
+                if resp.publicID() in responses:
+                    newInv.add(seiscomp.datamodel.ResponseFIR(resp))
+            if req._disconnected:  # pylint: disable=W0212
+                return None
+            for i in range(inv.responsePolynomialCount()):
+                resp = inv.responsePolynomial(i)
+                if resp.publicID() in responses:
+                    newInv.add(seiscomp.datamodel.ResponsePolynomial(resp))
+            if req._disconnected:  # pylint: disable=W0212
+                return None
+            for i in range(inv.responseFAPCount()):
+                resp = inv.responseFAP(i)
+                if resp.publicID() in responses:
+                    newInv.add(seiscomp.datamodel.ResponseFAP(resp))
+            if req._disconnected:  # pylint: disable=W0212
+                return None
+            for i in range(inv.responseIIRCount()):
+                resp = inv.responseIIR(i)
+                if resp.publicID() in responses:
+                    newInv.add(seiscomp.datamodel.ResponseIIR(resp))
+
+        return decCount
+
+
+# vim: ts=4 et

lib/python/seiscomp/fdsnws/utils.py

@@ -0,0 +1,201 @@
+################################################################################
+# Copyright (C) 2013-2014 gempa GmbH
+#
+# Common utility functions
+#
+# Author:  Stephan Herrnkind
+# Email:   herrnkind@gempa.de
+################################################################################
+
+import socket
+import traceback
+
+import twisted
+
+from twisted.internet import reactor, defer
+from twisted.python.failure import Failure
+from twisted.web import http
+
+
+import seiscomp.logging
+import seiscomp.core
+import seiscomp.io
+from seiscomp.client import Application
+
+twisted_version = (twisted.version.major, twisted.version.minor, twisted.version.micro)
+
+
+# -------------------------------------------------------------------------------
+# Converts a unicode string to a byte string
+def b_str(unicode_string):
+    return unicode_string.encode("utf-8", "replace")
+
+
+# -------------------------------------------------------------------------------
+# Converts a byte string to a unicode string
+def u_str(byte_string):
+    return byte_string.decode("utf-8", "replace")
+
+
+# -------------------------------------------------------------------------------
+# Tests if a SC3 inventory object is restricted
+def isRestricted(obj):
+    try:
+        return obj.restricted()
+    except ValueError:
+        return False
+
+
+# -------------------------------------------------------------------------------
+# Thread-safe write of string data using reactor main thread
+def writeTS(req, data):
+    reactor.callFromThread(req.write, b_str(data))
+
+
+# -------------------------------------------------------------------------------
+# Thread-safe write of binary data using reactor main thread
+def writeTSBin(req, data):
+    reactor.callFromThread(req.write, data)
+
+
+# -------------------------------------------------------------------------------
+# Finish requests deferred to threads
+def onFinish(result, req):
+    seiscomp.logging.debug(f"finish value = {str(result)}")
+    if isinstance(result, Failure):
+        err = result.value
+        if isinstance(err, defer.CancelledError):
+            seiscomp.logging.error("request canceled")
+            return
+        seiscomp.logging.error(
+            f"{result.getErrorMessage()} "
+            f"{traceback.format_tb(result.getTracebackObject())}"
+        )
+    else:
+        if result:
+            seiscomp.logging.debug("request successfully served")
+        else:
+            seiscomp.logging.debug("request failed")
+
+    reactor.callFromThread(req.finish)
+
+
+# -------------------------------------------------------------------------------
+# Handle connection errors
+def onCancel(failure, req):
+    if failure:
+        seiscomp.logging.error(
+            f"{failure.getErrorMessage()} "
+            f"{traceback.format_tb(failure.getTracebackObject())}"
+        )
+    else:
+        seiscomp.logging.error("request canceled")
+    req.cancel()
+
+
+# -------------------------------------------------------------------------------
+# Handle premature connection reset
+def onResponseFailure(_, call):
+    seiscomp.logging.error("response canceled")
+    call.cancel()
+
+
+# -------------------------------------------------------------------------------
+# Renders error page if the result set exceeds the configured maximum number
+# objects
+def accessLog(req, ro, code, length, err):
+    logger = Application.Instance()._accessLog  # pylint: disable=W0212
+    if logger is None:
+        return
+
+    logger.log(AccessLogEntry(req, ro, code, length, err))
+
+
+# -------------------------------------------------------------------------------
+# Compability function for stringToDatetime() change in Twisted 24.7, see
+# https://github.com/twisted/twisted/commit/731e370dfc5d2f7224dc1e12931ddf5c51b211a6
+def stringToDatetime(dateString):
+    if twisted_version < (24, 7):
+        return http.stringToDatetime(dateString)
+
+    # Since version 24.7 the argument needs to be a byte string
+    return http.stringToDatetime(dateString.encode("ascii"))
+
+
+################################################################################
+class Sink(seiscomp.io.ExportSink):
+    def __init__(self, request):
+        super().__init__()
+
+        self.request = request
+        self.written = 0
+
+    def write(self, data):
+        if self.request._disconnected:  # pylint: disable=W0212
+            return -1
+
+        writeTSBin(self.request, data)
+        self.written += len(data)
+        return len(data)
+
+
+################################################################################
+class AccessLogEntry:
+    def __init__(self, req, ro, code, length, err):
+        # user agent
+        agent = req.getHeader("User-Agent")
+        if agent is None:
+            agent = ""
+        else:
+            agent = agent[:100].replace("|", " ")
+
+        if err is None:
+            err = ""
+
+        service, user, accessTime, procTime = "", "", "", 0
+        net, sta, loc, cha = "", "", "", ""
+        if ro is not None:
+            # processing time in milliseconds
+            procTime = int((seiscomp.core.Time.GMT() - ro.accessTime).length() * 1000.0)
+
+            service = ro.service
+            if ro.userName is not None:
+                user = ro.userName
+            accessTime = str(ro.accessTime)
+
+            if ro.channel is not None:
+                if ro.channel.net is not None:
+                    net = ",".join(ro.channel.net)
+                if ro.channel.sta is not None:
+                    sta = ",".join(ro.channel.sta)
+                if ro.channel.loc is not None:
+                    loc = ",".join(ro.channel.loc)
+                if ro.channel.cha is not None:
+                    cha = ",".join(ro.channel.cha)
+
+        # The host name of the client is resolved in the __str__ method by the
+        # logging thread so that a long running DNS reverse lookup may not slow
+        # down the request
+        self.msgPrefix = f"{service}|{u_str(req.getRequestHostname())}|{accessTime}|"
+
+        xff = req.requestHeaders.getRawHeaders("x-forwarded-for")
+        if xff:
+            self.userIP = xff[0].split(",")[0].strip()
+        else:
+            self.userIP = req.getClientIP()
+
+        self.clientIP = req.getClientIP()
+        self.msgSuffix = (
+            f"|{self.clientIP}|{length}|{procTime}|{err}|{agent}|{code}|{user}|{net}"
+            f"|{sta}|{loc}|{cha}||"
+        )
+
+    def __str__(self):
+        try:
+            userHost = socket.gethostbyaddr(self.userIP)[0]
+        except socket.herror:
+            userHost = self.userIP
+        return self.msgPrefix + userHost + self.msgSuffix
+
+
+# vim: ts=4 et