diff --git a/libs/gempa/caps/3rd-party/libmseed/libmseed.h b/libs/gempa/caps/3rd-party/libmseed/libmseed.h new file mode 100644 index 0000000..839d1ed --- /dev/null +++ b/libs/gempa/caps/3rd-party/libmseed/libmseed.h @@ -0,0 +1,1571 @@ +/** ************************************************************************ + * @file libmseed.h + * + * Interface declarations for the miniSEED Library (libmseed). + * + * This file is part of the miniSEED Library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (C) 2025: + * @author Chad Trabant, EarthScope Data Services + ***************************************************************************/ + +#ifndef LIBMSEED_H +#define LIBMSEED_H 1 + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LIBMSEED_VERSION "3.1.11" //!< Library version +#define LIBMSEED_RELEASE "2025.303" //!< Library release date + +/** @defgroup io-functions File and URL I/O */ +/** @defgroup miniseed-record Record Handling */ +/** @defgroup trace-list Trace List */ +/** @defgroup data-selections Data Selections */ +/** @defgroup string-functions Source Identifiers */ +/** @defgroup extra-headers Extra Headers */ +/** @defgroup record-list Record List */ +/** @defgroup time-related Time definitions and functions */ +/** @defgroup logging Central Logging */ +/** @defgroup utility-functions General Utility Functions */ +/** @defgroup leapsecond Leap Second Handling */ + +/** @defgroup low-level Low level definitions + @brief The low-down, the nitty gritty, the basics */ +/** @defgroup memory-allocators Memory Allocators + @ingroup low-level */ +/** @defgroup encoding-values Data Encodings + @ingroup low-level */ +/** @defgroup byte-swap-flags Byte swap flags + @ingroup low-level */ +/** @defgroup return-values Return codes + @ingroup low-level */ +/** @defgroup control-flags Control flags + @ingroup low-level */ + +/* C99 standard headers */ +#include +#include +#include +#include +#include +#include +#include + +/** @def PRIsize_t + @brief A printf() macro for portably printing size_t values */ +#define PRIsize_t "zu" + +#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64) +#define LMP_WIN 1 +#endif + +/* Set platform specific features, Windows versus everything else */ +#if defined(LMP_WIN) +#include +#include +#include + +/* Re-define print conversion for size_t values */ +#undef PRIsize_t +#if defined(WIN64) || defined(_WIN64) +#define PRIsize_t "I64u" +#else +#define PRIsize_t "I32u" +#endif + +/* For MSVC 2012 and earlier define standard int types, otherwise use inttypes.h */ +#if defined(_MSC_VER) && _MSC_VER <= 1700 +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef signed short int int16_t; +typedef unsigned short int uint16_t; +typedef signed int int32_t; +typedef unsigned int uint32_t; +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#endif + +/* For MSVC define PRId64 and SCNd64 and alternate functions */ +#if defined(_MSC_VER) +#if !defined(PRId64) +#define PRId64 "I64d" +#endif +#if !defined(SCNd64) +#define SCNd64 "I64d" +#endif + +#define snprintf _snprintf +#define vsnprintf _vsnprintf +#define strcasecmp _stricmp +#define strncasecmp _strnicmp +#define strtoull _strtoui64 +#define fileno _fileno +#define fdopen _fdopen +#endif + +/* Extras needed for MinGW */ +#if defined(__MINGW32__) || defined(__MINGW64__) +#include + +#define _fseeki64 fseeko64 +#define _ftelli64 ftello64 + +#define fstat _fstat +#define stat _stat +#endif +#else +/* All other platforms */ +#include +#include +#endif + +#define MINRECLEN 40 //!< Minimum miniSEED record length supported +#define MAXRECLEN 10485760 //!< Maximum miniSEED record length supported (10MiB) +#define MAXRECLENv2 131172 //!< Maximum v2 miniSEED record length supported (131+ KiB or 2^17) + +#define LM_SIDLEN 64 //!< Length of source ID string + +/** @def MS_ISRATETOLERABLE + @brief Macro to test default sample rate tolerance: abs(1-sr1/sr2) < 0.0001 */ +#define MS_ISRATETOLERABLE(A, B) (fabs (1.0 - ((A) / (B))) < 0.0001) + +/** @def MS2_ISDATAINDICATOR + @brief Macro to test a character for miniSEED 2.x data record/quality indicators */ +#define MS2_ISDATAINDICATOR(X) ((X) == 'D' || (X) == 'R' || (X) == 'Q' || (X) == 'M') + +/** @def MS3_ISVALIDHEADER + * @hideinitializer + * Macro to test a buffer for a miniSEED 3.x data record signature by checking + * header values at known byte offsets: + * - 0 = "M" + * - 1 = "S" + * - 2 = 3 + * - 12 = valid hour (0-23) + * - 13 = valid minute (0-59) + * - 14 = valid second (0-60) + * + * Usage, X buffer must contain at least 15 bytes: + * @code + * MS3_ISVALIDHEADER ((char *)X) + * @endcode + */ +#define MS3_ISVALIDHEADER(X) \ + (*(X) == 'M' && *((X) + 1) == 'S' && *((X) + 2) == 3 && (uint8_t)(*((X) + 12)) >= 0 && \ + (uint8_t)(*((X) + 12)) <= 23 && (uint8_t)(*((X) + 13)) >= 0 && (uint8_t)(*((X) + 13)) <= 59 && \ + (uint8_t)(*((X) + 14)) >= 0 && (uint8_t)(*((X) + 14)) <= 60) + +/** @def MS2_ISVALIDHEADER + * @hideinitializer + * Macro to test a buffer for a miniSEED 2.x data record signature by checking + * header values at known byte offsets: + * - [0-5] = Digits, spaces or NULL, SEED sequence number + * - 6 = Data record quality indicator + * - 7 = Space or NULL [not valid SEED] + * - 24 = Start hour (0-23) + * - 25 = Start minute (0-59) + * - 26 = Start second (0-60) + * + * Usage, X buffer must contain at least 27 bytes: + * @code + * MS2_ISVALIDHEADER ((char *)X) + * @endcode + */ +#define MS2_ISVALIDHEADER(X) \ + ((isdigit ((uint8_t)*(X)) || *(X) == ' ' || !*(X)) && \ + (isdigit ((uint8_t)*((X) + 1)) || *((X) + 1) == ' ' || !*((X) + 1)) && \ + (isdigit ((uint8_t)*((X) + 2)) || *((X) + 2) == ' ' || !*((X) + 2)) && \ + (isdigit ((uint8_t)*((X) + 3)) || *((X) + 3) == ' ' || !*((X) + 3)) && \ + (isdigit ((uint8_t)*((X) + 4)) || *((X) + 4) == ' ' || !*((X) + 4)) && \ + (isdigit ((uint8_t)*((X) + 5)) || *((X) + 5) == ' ' || !*((X) + 5)) && \ + MS2_ISDATAINDICATOR (*((X) + 6)) && (*((X) + 7) == ' ' || *((X) + 7) == '\0') && \ + (uint8_t)(*((X) + 24)) >= 0 && (uint8_t)(*((X) + 24)) <= 23 && (uint8_t)(*((X) + 25)) >= 0 && \ + (uint8_t)(*((X) + 25)) <= 59 && (uint8_t)(*((X) + 26)) >= 0 && (uint8_t)(*((X) + 26)) <= 60) + +/** A simple bitwise AND test to return 0 or 1 */ +#define bit(x, y) ((x) & (y)) ? 1 : 0 + +/** Annotation for deprecated API components */ +#ifdef _MSC_VER +#define DEPRECATED __declspec (deprecated) +#elif defined(__GNUC__) | defined(__clang__) +#define DEPRECATED __attribute__ ((__deprecated__)) +#else +#define DEPRECATED +#endif + +/** @addtogroup time-related + @brief Definitions and functions for related to library time values + + Internally the library uses an integer value to represent time as + the number of nanoseconds since the Unix/POSIX epoch (Jan 1 1970). + + @{ */ + +/** @brief libmseed time type, integer nanoseconds since the Unix/POSIX epoch (00:00:00 Thursday, 1 + January 1970) + + This time scale can represent a range from before year 0 through mid-year 2262. +*/ +typedef int64_t nstime_t; + +/** @def NSTMODULUS + @brief Define the high precision time tick interval as 1/modulus seconds + corresponding to **nanoseconds**. **/ +#define NSTMODULUS 1000000000 + +/** @def NSTERROR + @brief Error code for routines that normally return a high precision time. + The time value corresponds to '1902-01-01T00:00:00.000000000Z'. **/ +#define NSTERROR -2145916800000000000LL + +/** @def NSTUNSET + @brief Special nstime_t value meaning "unset". + The time value corresponds to '1902-01-01T00:00:00.000000001Z'. **/ +#define NSTUNSET -2145916799999999999LL + +/** @def MS_EPOCH2NSTIME + @brief macro to convert Unix/POSIX epoch time to high precision epoch time */ +#define MS_EPOCH2NSTIME(X) (X) * (nstime_t)NSTMODULUS + +/** @def MS_NSTIME2EPOCH + @brief Macro to convert high precision epoch time to Unix/POSIX epoch time */ +#define MS_NSTIME2EPOCH(X) (X) / NSTMODULUS + +/** @def MS_HPTIME2NSTIME + @brief Convert a hptime_t value (used by previous releases) to nstime_t + + An HTPTIME/hptime_t value, used by libmseed major version <= 2, + defines microsecond ticks. An NSTIME/nstime_t value, used by this + version of the library, defines nanosecond ticks. +*/ +#define MS_HPTIME2NSTIME(X) (X) * (nstime_t)1000 + +/** @def MS_NSTIME2HPTIME + @brief Convert an nstime_t value to hptime_t (used by previous releases) + + An HTPTIME/hptime_t value, used by libmseed major version <= 2, + defines microsecond ticks. An NSTIME/nstime_t value, used by this + version of the library, defines nanosecond ticks. + */ +#define MS_NSTIME2HPTIME(X) (X) / 1000 + +/** @enum ms_timeformat_t + @brief Time format identifiers + + Formats values: + - \b ISOMONTHDAY - \c "YYYY-MM-DDThh:mm:ss.sssssssss", ISO 8601 in month-day format + - \b ISOMONTHDAY_Z - \c "YYYY-MM-DDThh:mm:ss.sssssssssZ", ISO 8601 in month-day format with + trailing Z + - \b ISOMONTHDAY_DOY - \c "YYYY-MM-DD hh:mm:ss.sssssssss (doy)", ISOMONTHDAY with day-of-year + - \b ISOMONTHDAY_DOY_Z - \c "YYYY-MM-DD hh:mm:ss.sssssssssZ (doy)", ISOMONTHDAY with day-of-year + and trailing Z + - \b ISOMONTHDAY_SPACE - \c "YYYY-MM-DD hh:mm:ss.sssssssss", same as ISOMONTHDAY with space + separator + - \b ISOMONTHDAY_SPACE_Z - \c "YYYY-MM-DD hh:mm:ss.sssssssssZ", same as ISOMONTHDAY with space + separator and trailing Z + - \b SEEDORDINAL - \c "YYYY,DDD,hh:mm:ss.sssssssss", SEED day-of-year format + - \b UNIXEPOCH - \c "ssssssssss.sssssssss", Unix epoch value + - \b NANOSECONDEPOCH - \c "sssssssssssssssssss", Nanosecond epoch value + */ +typedef enum +{ + ISOMONTHDAY = 0, + ISOMONTHDAY_Z = 1, + ISOMONTHDAY_DOY = 2, + ISOMONTHDAY_DOY_Z = 3, + ISOMONTHDAY_SPACE = 4, + ISOMONTHDAY_SPACE_Z = 5, + SEEDORDINAL = 6, + UNIXEPOCH = 7, + NANOSECONDEPOCH = 8 +} ms_timeformat_t; + +/** @enum ms_subseconds_t + @brief Subsecond format identifiers + + Formats values: + - \b NONE - No subseconds + - \b MICRO - Microsecond resolution + - \b NANO - Nanosecond resolution + - \b MICRO_NONE - Microsecond resolution if subseconds are non-zero, otherwise no subseconds + - \b NANO_NONE - Nanosecond resolution if subseconds are non-zero, otherwise no subseconds + - \b NANO_MICRO - Nanosecond resolution if there are sub-microseconds, otherwise microseconds + resolution + - \b NANO_MICRO_NONE - Nanosecond resolution if present, microsecond if present, otherwise no + subseconds + */ +typedef enum +{ + NONE = 0, + MICRO = 1, + NANO = 2, + MICRO_NONE = 3, + NANO_NONE = 4, + NANO_MICRO = 5, + NANO_MICRO_NONE = 6 +} ms_subseconds_t; + +extern int ms_nstime2time (nstime_t nstime, uint16_t *year, uint16_t *yday, uint8_t *hour, + uint8_t *min, uint8_t *sec, uint32_t *nsec); +extern char *ms_nstime2timestr_n (nstime_t nstime, char *timestr, size_t timestrsize, + ms_timeformat_t timeformat, ms_subseconds_t subsecond); +DEPRECATED extern char *ms_nstime2timestr (nstime_t nstime, char *timestr, + ms_timeformat_t timeformat, ms_subseconds_t subsecond); +DEPRECATED extern char *ms_nstime2timestrz (nstime_t nstime, char *timestr, + ms_timeformat_t timeformat, ms_subseconds_t subsecond); +extern nstime_t ms_time2nstime (int year, int yday, int hour, int min, int sec, uint32_t nsec); +extern nstime_t ms_timestr2nstime (const char *timestr); +extern nstime_t ms_mdtimestr2nstime (const char *timestr); +extern nstime_t ms_seedtimestr2nstime (const char *seedtimestr); +extern int ms_doy2md (int year, int yday, int *month, int *mday); +extern int ms_md2doy (int year, int month, int mday, int *yday); + +/** @} */ + +/** @page sample-types Sample Types + @brief Data sample types used by the library. + + Sample types are represented using a single character as follows: + - \c 't' - Text data samples + - \c 'i' - 32-bit integer data samples + - \c 'f' - 32-bit float (IEEE) data samples + - \c 'd' - 64-bit float (IEEE) data samples +*/ + +/** @def MS_PACK_DEFAULT_RECLEN + @brief Default record length to use when ::MS3Record.reclen == -1 + */ +#define MS_PACK_DEFAULT_RECLEN 4096 + +/** @def MS_PACK_DEFAULT_ENCODING + @brief Default data encoding to use when ::MS3Record.encoding == -1 + */ +#define MS_PACK_DEFAULT_ENCODING DE_STEIM2 + +/** @addtogroup miniseed-record + @brief Definitions and functions related to individual miniSEED records + @{ */ + +/** @brief miniSEED record container */ +typedef struct MS3Record +{ + const char *record; //!< Raw miniSEED record, if available + int32_t reclen; //!< Length of miniSEED record in bytes + uint8_t swapflag; //!< Byte swap indicator (bitmask), see @ref byte-swap-flags + + /* Common header fields in accessible form */ + char sid[LM_SIDLEN]; //!< Source identifier as URN, max length @ref LM_SIDLEN + uint8_t formatversion; //!< Format major version + uint8_t flags; //!< Record-level bit flags + nstime_t starttime; //!< Record start time (first sample) + double samprate; //!< Nominal sample rate as samples/second (Hz) or period (s) + int16_t encoding; //!< Data encoding format, see @ref encoding-values + uint8_t pubversion; //!< Publication version + int64_t samplecnt; //!< Number of samples in record + uint32_t crc; //!< CRC of entire record + uint16_t extralength; //!< Length of extra headers in bytes + uint32_t datalength; //!< Length of data payload in bytes + char *extra; //!< Pointer to extra headers + + /* Data sample fields */ + void *datasamples; //!< Data samples, \a numsamples of type \a sampletype + uint64_t datasize; //!< Size of datasamples buffer in bytes + int64_t numsamples; //!< Number of data samples in datasamples + char sampletype; //!< Sample type code: t, i, f, d @ref sample-types +} MS3Record; + +/** @def MS3Record_INITIALIZER + @brief Initialializer for a ::MS3Record */ +#define MS3Record_INITIALIZER \ + {.record = NULL, \ + .reclen = -1, \ + .swapflag = 0, \ + .sid = {0}, \ + .formatversion = 0, \ + .flags = 0, \ + .starttime = NSTUNSET, \ + .samprate = 0.0, \ + .encoding = -1, \ + .pubversion = 0, \ + .samplecnt = -1, \ + .crc = 0, \ + .extralength = 0, \ + .datalength = 0, \ + .extra = NULL, \ + .datasamples = NULL, \ + .datasize = 0, \ + .numsamples = 0, \ + .sampletype = 0} + +extern int msr3_parse (const char *record, uint64_t recbuflen, MS3Record **ppmsr, uint32_t flags, + int8_t verbose); + +extern int msr3_pack (const MS3Record *msr, void (*record_handler) (char *, int, void *), + void *handlerdata, int64_t *packedsamples, uint32_t flags, int8_t verbose); + +extern int msr3_repack_mseed3 (const MS3Record *msr, char *record, uint32_t recbuflen, + int8_t verbose); + +extern int msr3_repack_mseed2 (const MS3Record *msr, char *record, uint32_t recbuflen, + int8_t verbose); + +extern int msr3_pack_header3 (const MS3Record *msr, char *record, uint32_t recbuflen, + int8_t verbose); + +extern int msr3_pack_header2 (const MS3Record *msr, char *record, uint32_t recbuflen, + int8_t verbose); + +extern int64_t msr3_unpack_data (MS3Record *msr, int8_t verbose); + +extern int msr3_data_bounds (const MS3Record *msr, uint32_t *dataoffset, uint32_t *datasize); + +extern int64_t ms_decode_data (const void *input, uint64_t inputsize, uint8_t encoding, + uint64_t samplecount, void *output, uint64_t outputsize, + char *sampletype, int8_t swapflag, const char *sid, int8_t verbose); + +extern MS3Record *msr3_init (MS3Record *msr); +extern void msr3_free (MS3Record **ppmsr); +extern MS3Record *msr3_duplicate (const MS3Record *msr, int8_t datadup); +extern nstime_t msr3_endtime (const MS3Record *msr); +extern void msr3_print (const MS3Record *msr, int8_t details); +extern int msr3_resize_buffer (MS3Record *msr); +extern double msr3_sampratehz (const MS3Record *msr); +extern nstime_t msr3_nsperiod (const MS3Record *msr); +extern double msr3_host_latency (const MS3Record *msr); + +extern int64_t ms3_detect (const char *record, uint64_t recbuflen, uint8_t *formatversion); +extern int ms_parse_raw3 (const char *record, int maxreclen, int8_t details); +extern int ms_parse_raw2 (const char *record, int maxreclen, int8_t details, int8_t swapflag); +/** @} */ + +/** @addtogroup data-selections + @brief Data selections to be used as filters + + Selections are the identification of data, by source identifier + and time ranges, that are desired. Capability is included to read + selections from files and to match data against a selection list. + + For data to be selected it must only match one of the selection + entries. In other words, multiple selection entries are treated + with OR logic. + + The ms3_readmsr_selection() and ms3_readtracelist_selection() + routines accept ::MS3Selections and allow selective (and + efficient) reading of data from files. + @{ */ + +/** @brief Data selection structure time window definition containers */ +typedef struct MS3SelectTime +{ + nstime_t starttime; //!< Earliest data for matching channels, use ::NSTUNSET for open + nstime_t endtime; //!< Latest data for matching channels, use ::NSTUNSET for open + struct MS3SelectTime *next; //!< Pointer to next selection time, NULL if the last +} MS3SelectTime; + +/** @brief Data selection structure definition containers */ +typedef struct MS3Selections +{ + char sidpattern[100]; //!< Matching (globbing) pattern for source ID + struct MS3SelectTime *timewindows; //!< Pointer to time window list for this source ID + struct MS3Selections *next; //!< Pointer to next selection, NULL if the last + uint8_t pubversion; //!< Selected publication version, use 0 for any +} MS3Selections; + +extern const MS3Selections *ms3_matchselect (const MS3Selections *selections, const char *sid, + nstime_t starttime, nstime_t endtime, int pubversion, + const MS3SelectTime **ppselecttime); +extern const MS3Selections *msr3_matchselect (const MS3Selections *selections, const MS3Record *msr, + const MS3SelectTime **ppselecttime); +extern int ms3_addselect (MS3Selections **ppselections, const char *sidpattern, nstime_t starttime, + nstime_t endtime, uint8_t pubversion); +extern int ms3_addselect_comp (MS3Selections **ppselections, char *network, char *station, + char *location, char *channel, nstime_t starttime, nstime_t endtime, + uint8_t pubversion); +extern int ms3_readselectionsfile (MS3Selections **ppselections, const char *filename); +extern void ms3_freeselections (MS3Selections *selections); +extern void ms3_printselections (const MS3Selections *selections); +/** @} */ + +/** @addtogroup record-list + @{ */ + +/** @brief A miniSEED record pointer and metadata + * + * Used to construct a list of data records that contributed to a + * trace segment. + * + * The location of the record is identified at a memory address (\a + * bufferptr), the location in an open file (\a fileptr and \a + * fileoffset), or the location in a file (\a filename and \a + * fileoffset). + * + * A ::MS3Record is stored with and contains the bit flags, extra + * headers, etc. for the record. + * + * The \a dataoffset to the encoded data is stored to enable direct + * decoding of data samples without re-parsing the header, used by + * mstl3_unpack_recordlist(). + * + * Note: the list is stored in the time order that the entries + * contributed to the segment. + * + * @see mstl3_unpack_recordlist() + */ +typedef struct MS3RecordPtr +{ + const char *bufferptr; //!< Pointer in buffer to record, NULL if not used + FILE *fileptr; //!< Pointer to open FILE containing record, NULL if not used + const char *filename; //!< Pointer to file name containing record, NULL if not used + int64_t fileoffset; //!< Offset into file to record for \a fileptr or \a filename + MS3Record *msr; //!< Pointer to ::MS3Record for this record + nstime_t endtime; //!< End time of record, time of last sample + uint32_t dataoffset; //!< Offset from start of record to encoded data + void *prvtptr; //!< Private pointer, will not be populated by library but will be free'd + struct MS3RecordPtr *next; //!< Pointer to next entry, NULL if the last +} MS3RecordPtr; + +/** @brief Record list, holds ::MS3RecordPtr entries that contribute to a given ::MS3TraceSeg */ +typedef struct MS3RecordList +{ + uint64_t recordcnt; //!< Count of records in the list (for convenience) + MS3RecordPtr *first; //!< Pointer to first entry, NULL if the none + MS3RecordPtr *last; //!< Pointer to last entry, NULL if the none +} MS3RecordList; + +/** @} */ + +/** @addtogroup trace-list + @brief A container for continuous data + + Trace lists are a container to organize continuous segments of + data. By combining miniSEED data records into trace lists, the + time series is reconstructed and ready for processing, conversion, + summarization, etc. + + A trace list container starts with an ::MS3TraceList, which + contains one or more ::MS3TraceID entries, which each contain one + or more ::MS3TraceSeg entries. The ::MS3TraceID and ::MS3TraceSeg + entries are easily traversed as linked structures. + + The overall structure is illustrated as: + - MS3TraceList + - MS3TraceID + - MS3TraceSeg + - MS3TraceSeg + - ... + - MS3TraceID + - MS3TraceSeg + - MS3TraceSeg + - ... + - ... + + @note A trace list does not contain all of the details of a miniSEED + record. In particular details that are not relevant to represent the series + such as header flags, extra headers like event detections, etc. + + \sa ms3_readtracelist() + \sa ms3_readtracelist_timewin() + \sa ms3_readtracelist_selection() + \sa mstl3_writemseed() + @{ */ + +/** @brief Maximum skip list height for MSTraceIDs */ +#define MSTRACEID_SKIPLIST_HEIGHT 8 + +/** @brief Container for a continuous trace segment, linkable */ +typedef struct MS3TraceSeg +{ + nstime_t starttime; //!< Time of first sample + nstime_t endtime; //!< Time of last sample + double samprate; //!< Nominal sample rate (Hz) + int64_t samplecnt; //!< Number of samples in trace coverage + void *datasamples; //!< Data samples, \a numsamples of type \a sampletype + uint64_t datasize; //!< Size of datasamples buffer in bytes + int64_t numsamples; //!< Number of data samples in datasamples + char sampletype; //!< Sample type code, see @ref sample-types + void *prvtptr; //!< Private pointer for general use, unused by library unless ::MSF_PPUPDATETIME + //!< is set + struct MS3RecordList *recordlist; //!< List of pointers to records that contributed + struct MS3TraceSeg *prev; //!< Pointer to previous segment + struct MS3TraceSeg *next; //!< Pointer to next segment, NULL if the last +} MS3TraceSeg; + +/** @brief Container for a trace ID, linkable */ +typedef struct MS3TraceID +{ + char sid[LM_SIDLEN]; //!< Source identifier as URN, max length @ref LM_SIDLEN + uint8_t pubversion; //!< Largest contributing publication version + nstime_t earliest; //!< Time of earliest sample + nstime_t latest; //!< Time of latest sample + void *prvtptr; //!< Private pointer for general use, unused by library + uint32_t numsegments; //!< Number of segments for this ID + struct MS3TraceSeg *first; //!< Pointer to first of list of segments + struct MS3TraceSeg *last; //!< Pointer to last of list of segments + struct MS3TraceID + *next[MSTRACEID_SKIPLIST_HEIGHT]; //!< Next trace ID at first pointer, NULL if the last + uint8_t height; //!< Height of skip list at \a next +} MS3TraceID; + +/** @brief Container for a collection of continuous trace segment, linkable */ +typedef struct MS3TraceList +{ + uint32_t numtraceids; //!< Number of traces IDs in list + struct MS3TraceID traces; //!< Head node of trace skip list, first entry at \a traces.next[0] + uint64_t prngstate; //!< INTERNAL: State for Pseudo RNG +} MS3TraceList; + +/** @brief Callback functions that return time and sample rate tolerances + * + * A container for function pointers that return time and sample rate + * tolerances that are used for merging data into ::MS3TraceList + * containers. The functions are provided with a ::MS3Record and must + * return the acceptable tolerances to merge this with other data. + * + * The \c time(MS3Record) function must return a time tolerance in seconds. + * + * The \c samprate(MS3Record) function must return a sampling rate tolerance in Hertz. + * + * For any function pointer set to NULL a default tolerance will be used. + * + * Illustrated usage: + * @code + * MS3Tolerance tolerance; + * + * tolerance.time = my_time_tolerance_function; + * tolerance.samprate = my_samprate_tolerance_function; + * + * mstl3_addmsr (mstl, msr, 0, 1, &tolerance); + * @endcode + * + * \sa mstl3_addmsr() + */ +typedef struct MS3Tolerance +{ + double (*time) (const MS3Record *msr); //!< Pointer to function that returns time tolerance + double (*samprate) ( + const MS3Record *msr); //!< Pointer to function that returns sample rate tolerance +} MS3Tolerance; + +/** @def MS3Tolerance_INITIALIZER + @brief Initialializer for the tolerances ::MS3Tolerance */ +#define MS3Tolerance_INITIALIZER {.time = NULL, .samprate = NULL} + +extern MS3TraceList *mstl3_init (MS3TraceList *mstl); +extern void mstl3_free (MS3TraceList **ppmstl, int8_t freeprvtptr); +extern MS3TraceID *mstl3_findID (MS3TraceList *mstl, const char *sid, uint8_t pubversion, + MS3TraceID **prev); + +extern MS3TraceSeg *mstl3_addmsr (MS3TraceList *mstl, const MS3Record *msr, int8_t splitversion, + int8_t autoheal, uint32_t flags, const MS3Tolerance *tolerance); +extern MS3TraceSeg *mstl3_addmsr_recordptr (MS3TraceList *mstl, const MS3Record *msr, + MS3RecordPtr **pprecptr, int8_t splitversion, + int8_t autoheal, uint32_t flags, + const MS3Tolerance *tolerance); +extern int64_t mstl3_readbuffer (MS3TraceList **ppmstl, const char *buffer, uint64_t bufferlength, + int8_t splitversion, uint32_t flags, const MS3Tolerance *tolerance, + int8_t verbose); +extern int64_t mstl3_readbuffer_selection (MS3TraceList **ppmstl, const char *buffer, + uint64_t bufferlength, int8_t splitversion, + uint32_t flags, const MS3Tolerance *tolerance, + const MS3Selections *selections, int8_t verbose); +extern int64_t mstl3_unpack_recordlist (MS3TraceID *id, MS3TraceSeg *seg, void *output, + uint64_t outputsize, int8_t verbose); +extern int mstl3_convertsamples (MS3TraceSeg *seg, char type, int8_t truncate); +extern int mstl3_resize_buffers (MS3TraceList *mstl); +extern int64_t mstl3_pack (MS3TraceList *mstl, void (*record_handler) (char *, int, void *), + void *handlerdata, int reclen, int8_t encoding, int64_t *packedsamples, + uint32_t flags, int8_t verbose, char *extra); +extern int64_t mstl3_pack_ppupdate_flushidle (MS3TraceList *mstl, + void (*record_handler) (char *, int, void *), + void *handlerdata, int reclen, int8_t encoding, + int64_t *packedsamples, uint32_t flags, + int8_t verbose, char *extra, + uint32_t flush_idle_seconds); +extern int64_t mstl3_pack_segment (MS3TraceList *mstl, MS3TraceID *id, MS3TraceSeg *seg, + void (*record_handler) (char *, int, void *), void *handlerdata, + int reclen, int8_t encoding, int64_t *packedsamples, + uint32_t flags, int8_t verbose, char *extra); +DEPRECATED extern int64_t mstraceseg3_pack (MS3TraceID *, MS3TraceSeg *, + void (*) (char *, int, void *), void *, int, int8_t, + int64_t *, uint32_t, int8_t, char *); +extern void mstl3_printtracelist (const MS3TraceList *mstl, ms_timeformat_t timeformat, + int8_t details, int8_t gaps, int8_t versions); +extern void mstl3_printsynclist (const MS3TraceList *mstl, const char *dccid, + ms_subseconds_t subseconds); +extern void mstl3_printgaplist (const MS3TraceList *mstl, ms_timeformat_t timeformat, + double *mingap, double *maxgap); +/** @} */ + +/** @addtogroup io-functions + @brief Reading and writing interfaces for miniSEED to/from files or URLs + + The miniSEED reading interfaces read from either regular files or + URLs (if optional support is included). The miniSEED writing + interfaces write to regular files. + + URL support for reading is included by building the library with the + \b LIBMSEED_URL variable defined. URL path-specified resources can only be + read, e.g. HTTP GET requests. More advanced POST or form-based requests are + not supported. + + The function @ref libmseed_url_support() can be used as a run-time test + to determine if URL support is included in the library. + + Some parameters can be set that affect the reading of data from URLs, including: + - set the User-Agent header with @ref ms3_url_useragent() + - set username and password for authentication with @ref ms3_url_userpassword() + - set arbitrary headers with @ref ms3_url_addheader() + - disable TLS/SSL peer and host verficiation by setting \b LIBMSEED_SSL_NOVERIFY environment + variable + + Diagnostics: Setting environment variable \b LIBMSEED_URL_DEBUG enables detailed verbosity of + URL protocol exchanges. + + \sa ms3_readmsr() + \sa ms3_readmsr_selection() + \sa ms3_readtracelist() + \sa ms3_readtracelist_selection() + \sa msr3_writemseed() + \sa mstl3_writemseed() + @{ */ + +/** @brief Type definition for data source I/O: file-system versus URL */ +typedef struct LMIO +{ + enum + { + LMIO_NULL = 0, //!< IO handle type is undefined + LMIO_FILE = 1, //!< IO handle is FILE-type + LMIO_URL = 2, //!< IO handle is URL-type + LMIO_FD = 3 //!< IO handle is a provided file descriptor + } type; //!< IO handle type + void *handle; //!< Primary IO handle, either file or URL + void *handle2; //!< Secondary IO handle for URL + int still_running; //!< Fetch status flag for URL transmissions +} LMIO; + +/** @def LMIO_INITIALIZER + @brief Initialializer for the internal stream handle ::LMIO */ +#define LMIO_INITIALIZER {.type = LMIO_NULL, .handle = NULL, .handle2 = NULL, .still_running = 0} + +/** @brief State container for reading miniSEED records from files or URLs. + + In general these values should not be directly set or accessed. It is + possible to allocate a structure and set the \c path, \c startoffset, + and \c endoffset values for advanced usage. Note that file/URL start + and end offsets can also be parsed from the path name as well. +*/ +typedef struct MS3FileParam +{ + char path[512]; //!< INPUT: File name or URL + int64_t startoffset; //!< INPUT: Start position in input stream + int64_t endoffset; //!< INPUT: End position in input stream, 0 == unknown (e.g. pipe) + int64_t streampos; //!< OUTPUT: Read position of input stream + int64_t recordcount; //!< OUTPUT: Count of records read from this stream/file so far + + char *readbuffer; //!< INTERNAL: Read buffer, allocated internally + int readlength; //!< INTERNAL: Length of data in read buffer + int readoffset; //!< INTERNAL: Read offset in read buffer + uint32_t flags; //!< INTERNAL: Stream reading state flags + LMIO input; //!< INTERNAL: IO handle, file or URL +} MS3FileParam; + +/** @def MS3FileParam_INITIALIZER + @brief Initialializer for the internal file or URL I/O parameters ::MS3FileParam */ +#define MS3FileParam_INITIALIZER \ + {.path = "", \ + .startoffset = 0, \ + .endoffset = 0, \ + .streampos = 0, \ + .recordcount = 0, \ + .readbuffer = NULL, \ + .readlength = 0, \ + .readoffset = 0, \ + .flags = 0, \ + .input = LMIO_INITIALIZER} + +extern int ms3_readmsr (MS3Record **ppmsr, const char *mspath, uint32_t flags, int8_t verbose); +extern int ms3_readmsr_r (MS3FileParam **ppmsfp, MS3Record **ppmsr, const char *mspath, + uint32_t flags, int8_t verbose); +extern int ms3_readmsr_selection (MS3FileParam **ppmsfp, MS3Record **ppmsr, const char *mspath, + uint32_t flags, const MS3Selections *selections, int8_t verbose); +extern int ms3_readtracelist (MS3TraceList **ppmstl, const char *mspath, + const MS3Tolerance *tolerance, int8_t splitversion, uint32_t flags, + int8_t verbose); +extern int ms3_readtracelist_timewin (MS3TraceList **ppmstl, const char *mspath, + const MS3Tolerance *tolerance, nstime_t starttime, + nstime_t endtime, int8_t splitversion, uint32_t flags, + int8_t verbose); +extern int ms3_readtracelist_selection (MS3TraceList **ppmstl, const char *mspath, + const MS3Tolerance *tolerance, + const MS3Selections *selections, int8_t splitversion, + uint32_t flags, int8_t verbose); +extern int ms3_url_useragent (const char *program, const char *version); +extern int ms3_url_userpassword (const char *userpassword); +extern int ms3_url_addheader (const char *header); +extern void ms3_url_freeheaders (void); +extern int64_t msr3_writemseed (MS3Record *msr, const char *mspath, int8_t overwrite, + uint32_t flags, int8_t verbose); +extern int64_t mstl3_writemseed (MS3TraceList *mstl, const char *mspath, int8_t overwrite, + int maxreclen, int8_t encoding, uint32_t flags, int8_t verbose); +extern int libmseed_url_support (void); +extern MS3FileParam *ms3_mstl_init_fd (int fd); +/** @} */ + +/** @addtogroup string-functions + @brief Source identifier (SID) and string manipulation functions + + A source identifier uniquely identifies the generator of data in a + record. This is a small string, usually in the form of a URI. + For data identified with FDSN codes, the SID is usally a simple + combination of the codes. + + @{ */ +extern int ms_sid2nslc_n (const char *sid, char *net, size_t netsize, char *sta, size_t stasize, + char *loc, size_t locsize, char *chan, size_t chansize); +DEPRECATED extern int ms_sid2nslc (const char *sid, char *net, char *sta, char *loc, char *chan); +extern int ms_nslc2sid (char *sid, int sidlen, uint16_t flags, const char *net, const char *sta, + const char *loc, const char *chan); +extern int ms_seedchan2xchan (char *xchan, const char *seedchan); +extern int ms_xchan2seedchan (char *seedchan, const char *xchan); +extern int ms_strncpclean (char *dest, const char *source, int length); +extern int ms_strncpcleantail (char *dest, const char *source, int length); +extern int ms_strncpopen (char *dest, const char *source, int length); +/** @} */ + +/** @addtogroup extra-headers + @brief Structures and funtions to support extra headers + + Extra headers are stored as JSON within a data record header using + an anonymous, root object as a container for all extra headers. + For a full description consult the format specification. + + The library functions supporting extra headers allow specific + header identification using JSON Pointer identification. In this + notation each path element is an object until the final element + which is a key to specified header value. + + For example, a \a path specified as: + \code + "/objectA/objectB/header" + \endcode + + would correspond to the single JSON value in: + \code + { + "objectA": { + "objectB": { + "header":VALUE + } + } + } + \endcode + @{ */ + +/** + * @brief Container for event detection parameters for use in extra headers + * + * Actual values are optional, with special values indicating an unset + * state. + * + * @see mseh_add_event_detection_r + */ +typedef struct MSEHEventDetection +{ + char type[30]; /**< Detector type (e.g. "MURDOCK"), zero length = not included */ + char detector[30]; /**< Detector name, zero length = not included */ + double signalamplitude; /**< SignalAmplitude, 0.0 = not included */ + double signalperiod; /**< Signal period, 0.0 = not included */ + double backgroundestimate; /**< Background estimate, 0.0 = not included */ + char wave[30]; /**< Detection wave (e.g. "DILATATION"), zero length = not included */ + char units[30]; /**< Units of amplitude and background estimate (e.g. "COUNTS"), zero length = not + included */ + nstime_t onsettime; /**< Onset time, NSTUNSET = not included */ + uint8_t + medsnr[6]; /**< Signal to noise ratio for Murdock event detection, all zeros = not included */ + int medlookback; /**< Murdock event detection lookback value, -1 = not included */ + int medpickalgorithm; /**< Murdock event detection pick algoritm, -1 = not included */ + struct MSEHEventDetection *next; /**< Pointer to next, NULL if none */ +} MSEHEventDetection; + +/** + * @brief Container for calibration parameters for use in extra headers + * + * Actual values are optional, with special values indicating an unset + * state. + * + * @see mseh_add_calibration + */ +typedef struct MSEHCalibration +{ + char type[30]; /**< Calibration type (e.g. "STEP", "SINE", "PSEUDORANDOM"), zero length = not + included */ + nstime_t begintime; /**< Begin time, NSTUNSET = not included */ + nstime_t endtime; /**< End time, NSTUNSET = not included */ + int steps; /**< Number of step calibrations, -1 = not included */ + int firstpulsepositive; /**< Boolean, step cal. first pulse, -1 = not included */ + int alternatesign; /**< Boolean, step cal. alt. sign, -1 = not included */ + char trigger[30]; /**< Trigger, e.g. AUTOMATIC or MANUAL, zero length = not included */ + int continued; /**< Boolean, continued from prev. record, -1 = not included */ + double amplitude; /**< Amp. of calibration signal, 0.0 = not included */ + char inputunits[30]; /**< Units of input (e.g. volts, amps), zero length = not included */ + char amplituderange[30]; /**< E.g PEAKTOPTEAK, ZEROTOPEAK, RMS, RANDOM, zero length = not included + */ + double duration; /**< Duration in seconds, 0.0 = not included */ + double sineperiod; /**< Period of sine, 0.0 = not included */ + double stepbetween; /**< Interval bewteen steps, 0.0 = not included */ + char inputchannel[30]; /**< Channel of input, zero length = not included */ + double refamplitude; /**< Reference amplitude, 0.0 = not included */ + char coupling[30]; /**< Coupling, e.g. Resistive, Capacitive, zero length = not included */ + char rolloff[30]; /**< Rolloff of filters, zero length = not included */ + char noise[30]; /**< Noise for PR cals, e.g. White or Red, zero length = not included */ + struct MSEHCalibration *next; /**< Pointer to next, NULL if none */ +} MSEHCalibration; + +/** + * @brief Container for timing exception parameters for use in extra headers + * + * Actual values are optional, with special values indicating an unset + * state. + * + * @see mseh_add_timing_exception + */ +typedef struct MSEHTimingException +{ + nstime_t time; /**< Time of exception, NSTUNSET = not included */ + float vcocorrection; /**< VCO correction, from 0 to 100%, <0 = not included */ + int usec; /**< [DEPRECATED] microsecond time offset, 0 = not included */ + int receptionquality; /**< Reception quality, 0 to 100% clock accurracy, <0 = not included */ + uint32_t count; /**< The count thereof, 0 = not included */ + char type[16]; /**< E.g. "MISSING" or "UNEXPECTED", zero length = not included */ + char + clockstatus[128]; /**< Description of clock-specific parameters, zero length = not included */ +} MSEHTimingException; + +/** + * @brief Container for recenter parameters for use in extra headers + * + * Actual values are optional, with special values indicating an unset + * state. + * + * @see mseh_add_recenter + */ +typedef struct MSEHRecenter +{ + char type[30]; /**< Recenter type (e.g. "MASS", "GIMBAL"), zero length = not included */ + nstime_t begintime; /**< Begin time, NSTUNSET = not included */ + nstime_t endtime; /**< Estimated end time, NSTUNSET = not included */ + char trigger[30]; /**< Trigger, e.g. AUTOMATIC or MANUAL, zero length = not included */ +} MSEHRecenter; + +/** + * @brief Internal structure for holding parsed JSON extra headers. + * @see mseh_get_ptr_r() + * @see mseh_set_ptr_r() + */ +typedef struct LM_PARSED_JSON_s LM_PARSED_JSON; + +/** @def mseh_get + @brief A simple wrapper to access any type of extra header */ +#define mseh_get(msr, ptr, valueptr, type, maxlength) \ + mseh_get_ptr_r (msr, ptr, valueptr, type, maxlength, NULL) + +/** @def mseh_get_number + @brief A simple wrapper to access a number type extra header */ +#define mseh_get_number(msr, ptr, valueptr) mseh_get_ptr_r (msr, ptr, valueptr, 'n', 0, NULL) + +/** @def mseh_get_int64 + @brief A simple wrapper to access a number type extra header */ +#define mseh_get_int64(msr, ptr, valueptr) mseh_get_ptr_r (msr, ptr, valueptr, 'i', 0, NULL) + +/** @def mseh_get_string + @brief A simple wrapper to access a string type extra header */ +#define mseh_get_string(msr, ptr, buffer, maxlength) \ + mseh_get_ptr_r (msr, ptr, buffer, 's', maxlength, NULL) + +/** @def mseh_get_boolean + @brief A simple wrapper to access a boolean type extra header */ +#define mseh_get_boolean(msr, ptr, valueptr) mseh_get_ptr_r (msr, ptr, valueptr, 'b', 0, NULL) + +/** @def mseh_exists + @brief A simple wrapper to test existence of an extra header */ +#define mseh_exists(msr, ptr) (!mseh_get_ptr_r (msr, ptr, NULL, 0, 0, NULL)) + +extern int mseh_get_ptr_r (const MS3Record *msr, const char *ptr, void *value, char type, + uint32_t maxlength, LM_PARSED_JSON **parsestate); + +/** @def mseh_set + @brief A simple wrapper to set any type of extra header */ +#define mseh_set(msr, ptr, valueptr, type) mseh_set_ptr_r (msr, ptr, valueptr, type, NULL) + +/** @def mseh_set_number + @brief A simple wrapper to set a number type extra header */ +#define mseh_set_number(msr, ptr, valueptr) mseh_set_ptr_r (msr, ptr, valueptr, 'n', NULL) + +/** @def mseh_set_int64 + @brief A simple wrapper to set a number type extra header */ +#define mseh_set_int64(msr, ptr, valueptr) mseh_set_ptr_r (msr, ptr, valueptr, 'i', NULL) + +/** @def mseh_set_string + @brief A simple wrapper to set a string type extra header */ +#define mseh_set_string(msr, ptr, valueptr) mseh_set_ptr_r (msr, ptr, valueptr, 's', NULL) + +/** @def mseh_set_boolean + @brief A simple wrapper to set a boolean type extra header */ +#define mseh_set_boolean(msr, ptr, valueptr) mseh_set_ptr_r (msr, ptr, valueptr, 'b', NULL) + +extern int mseh_set_ptr_r (MS3Record *msr, const char *ptr, void *value, char type, + LM_PARSED_JSON **parsestate); + +extern int mseh_add_event_detection_r (MS3Record *msr, const char *ptr, + MSEHEventDetection *eventdetection, + LM_PARSED_JSON **parsestate); + +extern int mseh_add_calibration_r (MS3Record *msr, const char *ptr, MSEHCalibration *calibration, + LM_PARSED_JSON **parsestate); + +extern int mseh_add_timing_exception_r (MS3Record *msr, const char *ptr, + MSEHTimingException *exception, + LM_PARSED_JSON **parsestate); + +extern int mseh_add_recenter_r (MS3Record *msr, const char *ptr, MSEHRecenter *recenter, + LM_PARSED_JSON **parsestate); + +extern int mseh_serialize (MS3Record *msr, LM_PARSED_JSON **parsestate); +extern void mseh_free_parsestate (LM_PARSED_JSON **parsestate); +extern int mseh_replace (MS3Record *msr, char *jsonstring); + +extern int mseh_print (const MS3Record *msr, int indent); +/** @} */ + +/** @addtogroup record-list + @brief Functionality to build a list of records that contribute to a ::MS3TraceSeg + + As a @ref trace-list is constructed from data records, a list of + the records that contribute to each segment can be built by using + the ::MSF_RECORDLIST flag to @ref mstl3_readbuffer() and @ref + ms3_readtracelist(). Alternatively, a record list can be built by + adding records to a @ref trace-list using mstl3_addmsr_recordptr(). + + The main purpose of this functionality is to support an efficient, + 2-pass pattern of first reading a summary of data followed by + unpacking the samples. The unpacking can be performed selectively + on desired segments and optionally placed in a caller-supplied + buffer. + + The @ref mstl3_unpack_recordlist() function allows for the + unpacking of data samples for a given ::MS3TraceSeg into a + caller-specified buffer, or allocating the buffer if needed. + + \sa mstl3_readbuffer() + \sa mstl3_readbuffer_selection() + \sa ms3_readtracelist() + \sa ms3_readtracelist_selection() + \sa mstl3_unpack_recordlist() + \sa mstl3_addmsr_recordptr() +*/ + +/** @addtogroup logging + @brief Central logging functions for the library and calling programs + + This central logging facility is used for all logging performed by + the library. Calling programs may also wish to log messages via + the same facility for consistency. + + The logging can be configured to send messages to arbitrary + functions, referred to as \c log_print() and \c diag_print(). + This allows output to be re-directed to other logging systems if + needed. + + It is also possible to assign prefixes to log messages for + identification, referred to as \c logprefix and \c errprefix. + + @anchor logging-levels + Logging levels + -------------- + + Three message levels are recognized: + - 0 : Normal log messages, printed using \c log_print() with \c logprefix + - 1 : Diagnostic messages, printed using \c diag_print() with \c logprefix + - 2+ : Error messages, printed using \c diag_print() with \c errprefix + + It is the task of the ms_rlog() and ms_rlog_l() functions to + format a message using printf conventions and pass the formatted + string to the appropriate printing function. The convenience + macros ms_log() and ms_log_l() can be used to automatically set + the calling function name. + + @anchor log-registry + Log Registry + ------------ + + The log registry facility allows a calling program to disable + error (and warning) output from the library and either inspect it + or emit (print) as desired. + + By default log messages are sent directly to the printing + functions. Optionally, **error and warning messages** (levels 1 + and 2) can be accumulated in a log-registry. Verbose output + messages (level 0) are not accumulated in the registry. The + registry is enabled by setting the \c maxmessages argument of + either ms_rloginit() or ms_rloginit_l(). Messages can be emitted, + aka printed, using ms_rlog_emit() and cleared using + ms_rlog_free(). Alternatively, the ::MSLogRegistry associated + with a ::MSLogParam (or the global parameters at \c gMSLogParam). + + See \ref example-mseedview for a simple example of error and + warning message registry usage. + + @anchor log-threading + Logging in Threads + ------------------ + + By default the library is compiled in a mode where each thread of + a multi-threaded program will have it's own, default logging + parameters. __If you wish to change the default printing + functions, message prefixes, or enable the log registry, this must + be done per-thread.__ + + The library can be built with the \b LIBMSEED_NO_THREADING + variable defined, resulting in a mode where there are global + parameters for all threads. In general this should not be used + unless the system does not support the necessary thread-local + storage directives. + + @anchor MessageOnError + Message on Error + ---------------- + + Functions marked as \ref MessageOnError log a message when + returning an error status or logging a warning (log levels 1 and + 2). This indication can be useful when error and warning messages + are retained in \ref log-registry. + + @{ */ + +/** Maximum length of log messages in bytes */ +#define MAX_LOG_MSG_LENGTH 200 + +/** @brief Log registry entry. + \sa ms_rlog() + \sa ms_rlog_l() */ +typedef struct MSLogEntry +{ + int level; //!< Message level + char function[30]; //!< Function generating the message + char message[MAX_LOG_MSG_LENGTH]; //!< Log, warning or error message + struct MSLogEntry *next; +} MSLogEntry; + +/** @brief Log message registry. + \sa ms_rlog() + \sa ms_rlog_l() */ +typedef struct MSLogRegistry +{ + int maxmessages; + int messagecnt; + MSLogEntry *messages; +} MSLogRegistry; + +/** @def MSLogRegistry_INITIALIZER + @brief Initialializer for ::MSLogRegistry */ +#define MSLogRegistry_INITIALIZER {.maxmessages = 0, .messagecnt = 0, .messages = NULL} + +/** @brief Logging parameters. + __Callers should not modify these values directly and generally + should not need to access them.__ + + \sa ms_loginit() */ +typedef struct MSLogParam +{ + void (*log_print) (const char *); //!< Function to call for regular messages + const char *logprefix; //!< Message prefix for regular and diagnostic messages + void (*diag_print) (const char *); //!< Function to call for diagnostic and error messages + const char *errprefix; //!< Message prefix for error messages + MSLogRegistry registry; //!< Message registry +} MSLogParam; + +/** @def MSLogParam_INITIALIZER + @brief Initialializer for ::MSLogParam */ +#define MSLogParam_INITIALIZER \ + {.log_print = NULL, \ + .logprefix = NULL, \ + .diag_print = NULL, \ + .errprefix = NULL, \ + .registry = MSLogRegistry_INITIALIZER} + +/** @def ms_log + @brief Wrapper for ms_rlog(), call as __ms_log (level, format, ...)__ +*/ +#define ms_log(level, ...) ms_rlog (__func__, level, __VA_ARGS__) + +/** @def ms_log_l + @brief Wrapper for ms_rlog_l(), call as __ms_log_l (logp, level, format, ...)__ +*/ +#define ms_log_l(logp, level, ...) ms_rlog_l (logp, __func__, level, __VA_ARGS__) + +#if defined(__GNUC__) || defined(__clang__) +__attribute__ ((__format__ (__printf__, 3, 4))) +#endif +extern int +ms_rlog (const char *function, int level, const char *format, ...); +#if defined(__GNUC__) || defined(__clang__) +__attribute__ ((__format__ (__printf__, 4, 5))) +#endif +extern int +ms_rlog_l (MSLogParam *logp, const char *function, int level, const char *format, ...); + +/** @def ms_loginit + @brief Convenience wrapper for ms_rloginit(), omitting max messages, disabling registry */ +#define ms_loginit(log_print, logprefix, diag_print, errprefix) \ + ms_rloginit (log_print, logprefix, diag_print, errprefix, 0) + +/** @def ms_loginit_l + @brief Convenience wrapper for ms_rloginit_l(), omitting max messages, disabling registry */ +#define ms_loginit_l(logp, log_print, logprefix, diag_print, errprefix) \ + ms_rloginit_l (logp, log_print, logprefix, diag_print, errprefix, 0) + +extern void ms_rloginit (void (*log_print) (const char *), const char *logprefix, + void (*diag_print) (const char *), const char *errprefix, int maxmessages); +extern MSLogParam *ms_rloginit_l (MSLogParam *logp, void (*log_print) (const char *), + const char *logprefix, void (*diag_print) (const char *), + const char *errprefix, int maxmessages); +extern int ms_rlog_emit (MSLogParam *logp, int count, int context); +extern int ms_rlog_free (MSLogParam *logp); + +/** @} */ + +/** @addtogroup leapsecond + @brief Utilities for handling leap seconds + + @note The library contains an embedded list of leap seconds through + year 2023. These functions are only needed if leap seconds are added + in 2024 and beyond. + + The library contains functionality to load a list of leap seconds + into a global list, which is then used to determine when leap + seconds occurred, ignoring any flags in the data itself regarding + leap seconds. This is useful as past leap seconds are well known + and leap second indicators in data are, historically, more often + wrong than otherwise. + + The library uses the leap second list (and any flags in the data, + if no list is provided) to adjust the calculated time of the last + sample in a record. This allows proper merging of continuous + series generated through leap seconds. + + Normally, calling programs do not need to do any particular + handling of leap seconds after loading the leap second list. + + @note The library's internal, epoch-based time representation + cannot distinguish a leap second. On the epoch time scale a leap + second appears as repeat of the second that follows it, an + apparent duplicated second. Since the library does not know if + this value is a leap second or not, when converted to a time + string, the non-leap second representation is used, i.e. no second + values of "60" are generated. + + @{ */ +/** @brief Leap second list container */ +typedef struct LeapSecond +{ + nstime_t leapsecond; //!< Time of leap second as epoch since 1 January 1900 + int32_t TAIdelta; //!< TAI-UTC difference in seconds + struct LeapSecond *next; //!< Pointer to next entry, NULL if the last +} LeapSecond; + +/** Global leap second list */ +extern LeapSecond *leapsecondlist; +extern int ms_readleapseconds (const char *envvarname); +extern int ms_readleapsecondfile (const char *filename); +/** @} */ + +/** @addtogroup utility-functions + @brief General utilities + @{ */ + +extern uint8_t ms_samplesize (char sampletype); +extern int ms_encoding_sizetype (uint8_t encoding, uint8_t *samplesize, char *sampletype); +extern const char *ms_encodingstr (uint8_t encoding); +extern const char *ms_errorstr (int errorcode); + +extern nstime_t ms_sampletime (nstime_t time, int64_t offset, double samprate); +extern int ms_bigendianhost (void); + +/** DEPRECATED legacy implementation of fabs(), now a macro */ +#define ms_dabs(val) fabs (val) + +/** Portable version of POSIX ftello() to get file position in large files */ +extern int64_t lmp_ftell64 (FILE *stream); +/** Portable version of POSIX fseeko() to set position in large files */ +extern int lmp_fseek64 (FILE *stream, int64_t offset, int whence); +/** Portable version of POSIX nanosleep() to sleep for nanoseconds */ +extern uint64_t lmp_nanosleep (uint64_t nanoseconds); +/** Portable function to return the current system time */ +extern nstime_t lmp_systemtime (void); + +/** Return CRC32C value of supplied buffer, with optional starting CRC32C value */ +extern uint32_t ms_crc32c (const uint8_t *input, int length, uint32_t previousCRC32C); + +/** In-place byte swapping of 2 byte quantity */ +static inline void +ms_gswap2 (void *data2) +{ + uint16_t dat; + + memcpy (&dat, data2, 2); + + dat = ((dat & 0xff00) >> 8) | ((dat & 0x00ff) << 8); + + memcpy (data2, &dat, 2); +} + +/** In-place byte swapping of 4 byte quantity */ +static inline void +ms_gswap4 (void *data4) +{ + uint32_t dat; + + memcpy (&dat, data4, 4); + + dat = ((dat & 0xff000000) >> 24) | ((dat & 0x000000ff) << 24) | ((dat & 0x00ff0000) >> 8) | + ((dat & 0x0000ff00) << 8); + + memcpy (data4, &dat, 4); +} + +/** In-place byte swapping of 8 byte quantity */ +static inline void +ms_gswap8 (void *data8) +{ + uint64_t dat; + + memcpy (&dat, data8, sizeof (uint64_t)); + + dat = ((dat & 0xff00000000000000) >> 56) | ((dat & 0x00000000000000ff) << 56) | + ((dat & 0x00ff000000000000) >> 40) | ((dat & 0x000000000000ff00) << 40) | + ((dat & 0x0000ff0000000000) >> 24) | ((dat & 0x0000000000ff0000) << 24) | + ((dat & 0x000000ff00000000) >> 8) | ((dat & 0x00000000ff000000) << 8); + + memcpy (data8, &dat, sizeof (uint64_t)); +} + +#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) || defined(__clang__) +/** Deprecated: In-place byte swapping of 2 byte, memory-aligned, quantity */ +__attribute__ ((deprecated ("Use ms_gswap2 instead."))) static inline void +ms_gswap2a (void *data2) +{ + ms_gswap2 (data2); +} +/** Deprecated: In-place byte swapping of 4 byte, memory-aligned, quantity */ +__attribute__ ((deprecated ("Use ms_gswap4 instead."))) static inline void +ms_gswap4a (void *data4) +{ + ms_gswap4 (data4); +} +/** Deprecated: In-place byte swapping of 8 byte, memory-aligned, quantity */ +__attribute__ ((deprecated ("Use ms_gswap8 instead."))) static inline void +ms_gswap8a (void *data8) +{ + ms_gswap8 (data8); +} +#elif defined(_MSC_FULL_VER) && (_MSC_FULL_VER > 140050320) +/** Deprecated: In-place byte swapping of 2 byte, memory-aligned, quantity */ +__declspec (deprecated ("Use ms_gswap2 instead.")) static inline void +ms_gswap2a (void *data2) +{ + ms_gswap2 (data2); +} +/** Deprecated: In-place byte swapping of 4 byte, memory-aligned, quantity */ +__declspec (deprecated ("Use ms_gswap4 instead.")) static inline void +ms_gswap4a (void *data4) +{ + ms_gswap4 (data4); +} +/** Deprecated: In-place byte swapping of 8 byte, memory-aligned, quantity */ +__declspec (deprecated ("Use ms_gswap8 instead.")) static inline void +ms_gswap8a (void *data8) +{ + ms_gswap8 (data8); +} +#else +/** Deprecated: In-place byte swapping of 2 byte, memory-aligned, quantity */ +static inline void +ms_gswap2a (void *data2) +{ + ms_gswap2 (data2); +} +/** Deprecated: In-place byte swapping of 4 byte, memory-aligned, quantity */ +static inline void +ms_gswap4a (void *data4) +{ + ms_gswap4 (data4); +} +/** Deprecated: In-place byte swapping of 8 byte, memory-aligned, quantity */ +static inline void +ms_gswap8a (void *data8) +{ + ms_gswap8 (data8); +} +#endif + +/** @} */ + +/** Single byte flag type, for legacy use */ +typedef int8_t flag; + +/** @addtogroup memory-allocators + @brief User-definable memory allocators used by library + + The global structure \b libmseed_memory contains three function + pointers that are used for all memory allocation and freeing done + by the library. + + The following function pointers are available: + - \b libmseed_memory.malloc - requires a malloc()-like function + - \b libmseed_memory.realloc - requires a realloc()-like function + - \b libmseed_memory.free - requires a free()-like function + + By default the system malloc(), realloc(), and free() are used. + Equivalent to setting: + + \code + libmseed_memory.malloc = malloc; + libmseed_memory.realloc = realloc; + libmseed_memory.free = free; + \endcode + + @{ */ + +/** Container for memory management function pointers */ +typedef struct LIBMSEED_MEMORY +{ + void *(*malloc) (size_t); //!< Pointer to desired malloc() + void *(*realloc) (void *, size_t); //!< Pointer to desired realloc() + void (*free) (void *); //!< Pointer to desired free() +} LIBMSEED_MEMORY; + +/** Global memory management function list */ +extern LIBMSEED_MEMORY libmseed_memory; + +/** Global pre-allocation block size. + * + * When non-zero, memory re-allocations will be increased in blocks of this + * size. This is useful on platforms where the system realloc() is slow + * such as Windows. + * + * Default on Windows is 1 MiB, otherwise disabled. + * + * Set to 0 to disable pre-allocation. + * + * @see msr3_resize_buffer + * @see mstl3_resize_buffers + */ +extern size_t libmseed_prealloc_block_size; + +/** Internal realloc() wrapper that allocates in ::libmseed_prealloc_block_size blocks + * + * Preallocation is used by default on Windows and disabled otherwise. + * */ +extern void *libmseed_memory_prealloc (void *ptr, size_t size, size_t *currentsize); + +/** @} */ + +#define DE_ASCII DE_TEXT //!< Mapping of legacy DE_ASCII to DE_TEXT + +/** @addtogroup encoding-values + @brief Data encoding type defines + + These are FDSN-defined miniSEED data encoding values. The value + of ::MS3Record.encoding is set to one of these. These values may + be used anywhere and encoding value is needed. + + @{ */ +#define DE_TEXT 0 //!< Text encoding (UTF-8) +#define DE_INT16 1 //!< 16-bit integer +#define DE_INT32 3 //!< 32-bit integer +#define DE_FLOAT32 4 //!< 32-bit float (IEEE) +#define DE_FLOAT64 5 //!< 64-bit float (IEEE) +#define DE_STEIM1 10 //!< Steim-1 compressed integers +#define DE_STEIM2 11 //!< Steim-2 compressed integers +#define DE_GEOSCOPE24 12 //!< [Legacy] GEOSCOPE 24-bit integer +#define DE_GEOSCOPE163 13 //!< [Legacy] GEOSCOPE 16-bit gain ranged, 3-bit exponent +#define DE_GEOSCOPE164 14 //!< [Legacy] GEOSCOPE 16-bit gain ranged, 4-bit exponent +#define DE_CDSN 16 //!< [Legacy] CDSN 16-bit gain ranged +#define DE_SRO 30 //!< [Legacy] SRO 16-bit gain ranged +#define DE_DWWSSN 32 //!< [Legacy] DWWSSN 16-bit gain ranged +/** @} */ + +/** @addtogroup byte-swap-flags + @brief Flags indicating whether the header or payload needed byte swapping + + These are bit flags normally used to set/test the ::MS3Record.swapflag value. + + @{ */ +#define MSSWAP_HEADER 0x01 //!< Header needed byte swapping +#define MSSWAP_PAYLOAD 0x02 //!< Data payload needed byte swapping +/** @} */ + +/** @addtogroup return-values + @brief Common error codes returned by library functions. Error values will always be negative. + + \sa ms_errorstr() + @{ */ +#define MS_ENDOFFILE 1 //!< End of file reached return value +#define MS_NOERROR 0 //!< No error +#define MS_GENERROR -1 //!< Generic unspecified error +#define MS_NOTSEED -2 //!< Data not SEED +#define MS_WRONGLENGTH -3 //!< Length of data read was not correct +#define MS_OUTOFRANGE -4 //!< SEED record length out of range +#define MS_UNKNOWNFORMAT -5 //!< Unknown data encoding format +#define MS_STBADCOMPFLAG -6 //!< Steim, invalid compression flag(s) +#define MS_INVALIDCRC -7 //!< Invalid CRC +/** @} */ + +/** @addtogroup control-flags + @brief Parsing, packing and trace construction control flags + + These are bit flags that can be combined into a bitmask to control + aspects of the library's parsing, packing and trace managment routines. + + @{ */ +#define MSF_UNPACKDATA 0x0001 //!< [Parsing] Unpack data samples +#define MSF_SKIPNOTDATA 0x0002 //!< [Parsing] Skip input that cannot be identified as miniSEED +#define MSF_VALIDATECRC 0x0004 //!< [Parsing] Validate CRC (if version 3) +#define MSF_PNAMERANGE 0x0008 //!< [Parsing] Parse and utilize byte range from path name suffix +#define MSF_ATENDOFFILE 0x0010 //!< [Parsing] Reading routine is at the end of the file +#define MSF_SEQUENCE 0x0020 //!< [Packing] UNSUPPORTED: Maintain a record-level sequence number +#define MSF_FLUSHDATA \ + 0x0040 //!< [Packing] Pack all available data even if final record would not be filled +#define MSF_PACKVER2 0x0080 //!< [Packing] Pack as miniSEED version 2 instead of 3 +#define MSF_RECORDLIST 0x0100 //!< [TraceList] Build a ::MS3RecordList for each ::MS3TraceSeg +#define MSF_MAINTAINMSTL 0x0200 //!< [TraceList] Do not modify a trace list when packing +#define MSF_PPUPDATETIME \ + 0x0400 //!< [TraceList] Store update time (as nstime_t) at ::MS3TraceSeg.prvtptr +#define MSF_SPLITISVERSION \ + 0x0800 //!< [TraceList] Use the splitversion value as version instead of record version +#define MSF_SKIPADJACENTDUPLICATES 0x1000 //!< [TraceList] Skip adjacent duplicate records +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* LIBMSEED_H */ diff --git a/libs/gempa/caps/3rd-party/libmseed/mseedformat.h b/libs/gempa/caps/3rd-party/libmseed/mseedformat.h new file mode 100644 index 0000000..daf9612 --- /dev/null +++ b/libs/gempa/caps/3rd-party/libmseed/mseedformat.h @@ -0,0 +1,661 @@ +/*************************************************************************** + * Documentation and helpers for miniSEED structures. + * + * This file is part of the miniSEED Library. + * + * Copyright (c) 2024 Chad Trabant, EarthScope Data Services + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ***************************************************************************/ + +#ifndef MSEEDFORMAT_H +#define MSEEDFORMAT_H 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "libmseed.h" + +/* Length of Fixed Section of Data Header for miniSEED v3 */ +#define MS3FSDH_LENGTH 40 + +/*************************************************************************** + * miniSEED 3.0 Fixed Section of Data Header + * 40 bytes, plus length of identifier, plus length of extra headers + * + * # FIELD TYPE OFFSET + * 1 record indicator char[2] 0 + * 2 format version uint8_t 2 + * 3 flags uint8_t 3 + * 4a nanosecond uint32_t 4 + * 4b year uint16_t 8 + * 4c day uint16_t 10 + * 4d hour uint8_t 12 + * 4e min uint8_t 13 + * 4f sec uint8_t 14 + * 5 data encoding uint8_t 15 + * 6 sample rate/period float64 16 + * 7 number of samples uint32_t 24 + * 8 CRC of record uint32_t 28 + * 9 publication version uint8_t 32 + * 10 length of identifer uint8_t 33 + * 11 length of extra headers uint16_t 34 + * 12 length of data payload uint32_t 36 + * 13 source identifier char 40 + * 14 extra headers char 40 + field 10 + * 15 data payload encoded 40 + field 10 + field 11 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS3FSDH_INDICATOR(record) ((char*)record) +#define pMS3FSDH_FORMATVERSION(record) ((uint8_t*)((uint8_t*)record+2)) +#define pMS3FSDH_FLAGS(record) ((uint8_t*)((uint8_t*)record+3)) +#define pMS3FSDH_NSEC(record) ((uint32_t*)((uint8_t*)record+4)) +#define pMS3FSDH_YEAR(record) ((uint16_t*)((uint8_t*)record+8)) +#define pMS3FSDH_DAY(record) ((uint16_t*)((uint8_t*)record+10)) +#define pMS3FSDH_HOUR(record) ((uint8_t*)((uint8_t*)record+12)) +#define pMS3FSDH_MIN(record) ((uint8_t*)((uint8_t*)record+13)) +#define pMS3FSDH_SEC(record) ((uint8_t*)((uint8_t*)record+14)) +#define pMS3FSDH_ENCODING(record) ((uint8_t*)((uint8_t*)record+15)) +#define pMS3FSDH_SAMPLERATE(record) ((double*)((uint8_t*)record+16)) +#define pMS3FSDH_NUMSAMPLES(record) ((uint32_t*)((uint8_t*)record+24)) +#define pMS3FSDH_CRC(record) ((uint32_t*)((uint8_t*)record+28)) +#define pMS3FSDH_PUBVERSION(record) ((uint8_t*)((uint8_t*)record+32)) +#define pMS3FSDH_SIDLENGTH(record) ((uint8_t*)((uint8_t*)record+33)) +#define pMS3FSDH_EXTRALENGTH(record) ((uint16_t*)((uint8_t*)record+34)) +#define pMS3FSDH_DATALENGTH(record) ((uint32_t*)((uint8_t*)record+36)) +#define pMS3FSDH_SID(record) ((char*)((uint8_t*)record+40)) + +/* Length of Fixed Section of Data Header for miniSEED v2 */ +#define MS2FSDH_LENGTH 48 + +/*************************************************************************** + * miniSEED 2.4 Fixed Section of Data Header + * 48 bytes total + * + * FIELD TYPE OFFSET + * sequence_number char[6] 0 + * dataquality char 6 + * reserved char 7 + * station char[5] 8 + * location char[2] 13 + * channel char[3] 15 + * network char[2] 18 + * year uint16_t 20 + * day uint16_t 22 + * hour uint8_t 24 + * min uint8_t 25 + * sec uint8_t 26 + * unused uint8_t 27 + * fract uint16_t 28 + * numsamples uint16_t 30 + * samprate_fact int16_t 32 + * samprate_mult int16_t 34 + * act_flags uint8_t 36 + * io_flags uint8_t 37 + * dq_flags uint8_t 38 + * numblockettes uint8_t 39 + * time_correct int32_t 40 + * data_offset uint16_t 44 + * blockette_offset uint16_t 46 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2FSDH_SEQNUM(record) ((char*)record) +#define pMS2FSDH_DATAQUALITY(record) ((char*)((uint8_t*)record+6)) +#define pMS2FSDH_RESERVED(record) ((char*)((uint8_t*)record+7)) +#define pMS2FSDH_STATION(record) ((char*)((uint8_t*)record+8)) +#define pMS2FSDH_LOCATION(record) ((char*)((uint8_t*)record+13)) +#define pMS2FSDH_CHANNEL(record) ((char*)((uint8_t*)record+15)) +#define pMS2FSDH_NETWORK(record) ((char*)((uint8_t*)record+18)) +#define pMS2FSDH_YEAR(record) ((uint16_t*)((uint8_t*)record+20)) +#define pMS2FSDH_DAY(record) ((uint16_t*)((uint8_t*)record+22)) +#define pMS2FSDH_HOUR(record) ((uint8_t*)((uint8_t*)record+24)) +#define pMS2FSDH_MIN(record) ((uint8_t*)((uint8_t*)record+25)) +#define pMS2FSDH_SEC(record) ((uint8_t*)((uint8_t*)record+26)) +#define pMS2FSDH_UNUSED(record) ((uint8_t*)((uint8_t*)record+27)) +#define pMS2FSDH_FSEC(record) ((uint16_t*)((uint8_t*)record+28)) +#define pMS2FSDH_NUMSAMPLES(record) ((uint16_t*)((uint8_t*)record+30)) +#define pMS2FSDH_SAMPLERATEFACT(record) ((int16_t*)((uint8_t*)record+32)) +#define pMS2FSDH_SAMPLERATEMULT(record) ((int16_t*)((uint8_t*)record+34)) +#define pMS2FSDH_ACTFLAGS(record) ((uint8_t*)((uint8_t*)record+36)) +#define pMS2FSDH_IOFLAGS(record) ((uint8_t*)((uint8_t*)record+37)) +#define pMS2FSDH_DQFLAGS(record) ((uint8_t*)((uint8_t*)record+38)) +#define pMS2FSDH_NUMBLOCKETTES(record) ((uint8_t*)((uint8_t*)record+39)) +#define pMS2FSDH_TIMECORRECT(record) ((int32_t*)((uint8_t*)record+40)) +#define pMS2FSDH_DATAOFFSET(record) ((uint16_t*)((uint8_t*)record+44)) +#define pMS2FSDH_BLOCKETTEOFFSET(record) ((uint16_t*)((uint8_t*)record+46)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 100 - sample rate + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * samprate float 4 + * flags uint8_t 8 + * reserved uint8_t[3] 9 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B100_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B100_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B100_SAMPRATE(blockette) ((float*)((uint8_t*)blockette+4)) +#define pMS2B100_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+8)) +#define pMS2B100_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+9)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 200 - generic event detection + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * amplitude float 4 + * period float 8 + * background_est float 12 + * flags uint8_t 16 + * reserved uint8_t 17 + * year uint16_t 18 + * day uint16_t 20 + * hour uint8_t 22 + * min uint8_t 23 + * sec uint8_t 24 + * unused uint8_t 25 + * fract uint16_t 26 + * detector char[24] 28 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B200_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B200_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B200_AMPLITUDE(blockette) ((float*)((uint8_t*)blockette+4)) +#define pMS2B200_PERIOD(blockette) ((float*)((uint8_t*)blockette+8)) +#define pMS2B200_BACKGROUNDEST(blockette) ((float*)((uint8_t*)blockette+12)) +#define pMS2B200_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+16)) +#define pMS2B200_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+17)) +#define pMS2B200_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+18)) +#define pMS2B200_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+20)) +#define pMS2B200_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+22)) +#define pMS2B200_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+23)) +#define pMS2B200_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+24)) +#define pMS2B200_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+25)) +#define pMS2B200_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+26)) +#define pMS2B200_DETECTOR(blockette) ((char*)((uint8_t*)blockette+28)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 201 - Murdock event detection + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * amplitude float 4 + * period float 8 + * background_est float 12 + * flags uint8_t 16 + * reserved uint8_t 17 + * year uint16_t 18 + * day uint16_t 20 + * hour uint8_t 22 + * min uint8_t 23 + * sec uint8_t 24 + * unused uint8_t 25 + * fract uint16_t 26 + * snr_values uint8_t[6] 28 + * loopback uint8_t 34 + * pick_algorithm uint8_t 35 + * detector char[24] 36 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B201_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B201_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B201_AMPLITUDE(blockette) ((float*)((uint8_t*)blockette+4)) +#define pMS2B201_PERIOD(blockette) ((float*)((uint8_t*)blockette+8)) +#define pMS2B201_BACKGROUNDEST(blockette) ((float*)((uint8_t*)blockette+12)) +#define pMS2B201_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+16)) +#define pMS2B201_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+17)) +#define pMS2B201_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+18)) +#define pMS2B201_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+20)) +#define pMS2B201_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+22)) +#define pMS2B201_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+23)) +#define pMS2B201_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+24)) +#define pMS2B201_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+25)) +#define pMS2B201_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+26)) +#define pMS2B201_MEDSNR(blockette) ((uint8_t*)((uint8_t*)blockette+28)) +#define pMS2B201_LOOPBACK(blockette) ((uint8_t*)((uint8_t*)blockette+34)) +#define pMS2B201_PICKALGORITHM(blockette) ((uint8_t*)((uint8_t*)blockette+35)) +#define pMS2B201_DETECTOR(blockette) ((char*)((uint8_t*)blockette+36)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 300 - step calibration + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * year uint16_t 4 + * day uint16_t 6 + * hour uint8_t 8 + * min uint8_t 9 + * sec uint8_t 10 + * unused uint8_t 11 + * fract uint16_t 12 + * num calibrations uint8_t 14 + * flags uint8_t 15 + * step duration uint32_t 16 + * interval duration uint32_t 20 + * amplitude float 24 + * input channel char[3] 28 + * reserved uint8_t 31 + * reference amplitude uint32_t 32 + * coupling char[12] 36 + * rolloff char[12] 48 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B300_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B300_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B300_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+4)) +#define pMS2B300_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+6)) +#define pMS2B300_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+8)) +#define pMS2B300_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+9)) +#define pMS2B300_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+10)) +#define pMS2B300_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+11)) +#define pMS2B300_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+12)) +#define pMS2B300_NUMCALIBRATIONS(blockette) ((uint8_t*)((uint8_t*)blockette+14)) +#define pMS2B300_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+15)) +#define pMS2B300_STEPDURATION(blockette) ((uint32_t*)((uint8_t*)blockette+16)) +#define pMS2B300_INTERVALDURATION(blockette) ((uint32_t*)((uint8_t*)blockette+20)) +#define pMS2B300_AMPLITUDE(blockette) ((float*)((uint8_t*)blockette+24)) +#define pMS2B300_INPUTCHANNEL(blockette) ((char *)((uint8_t*)blockette+28)) +#define pMS2B300_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+31)) +#define pMS2B300_REFERENCEAMPLITUDE(blockette) ((uint32_t*)((uint8_t*)blockette+32)) +#define pMS2B300_COUPLING(blockette) ((char*)((uint8_t*)blockette+36)) +#define pMS2B300_ROLLOFF(blockette) ((char*)((uint8_t*)blockette+48)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 310 - sine calibration + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * year uint16_t 4 + * day uint16_t 6 + * hour uint8_t 8 + * min uint8_t 9 + * sec uint8_t 10 + * unused uint8_t 11 + * fract uint16_t 12 + * reserved1 uint8_t 14 + * flags uint8_t 15 + * duration uint32_t 16 + * period float 20 + * amplitude float 24 + * input channel char[3] 28 + * reserved2 uint8_t 31 + * reference amplitude uint32_t 32 + * coupling char[12] 36 + * rolloff char[12] 48 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B310_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B310_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B310_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+4)) +#define pMS2B310_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+6)) +#define pMS2B310_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+8)) +#define pMS2B310_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+9)) +#define pMS2B310_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+10)) +#define pMS2B310_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+11)) +#define pMS2B310_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+12)) +#define pMS2B310_RESERVED1(blockette) ((uint8_t*)((uint8_t*)blockette+14)) +#define pMS2B310_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+15)) +#define pMS2B310_DURATION(blockette) ((uint32_t*)((uint8_t*)blockette+16)) +#define pMS2B310_PERIOD(blockette) ((float*)((uint8_t*)blockette+20)) +#define pMS2B310_AMPLITUDE(blockette) ((float*)((uint8_t*)blockette+24)) +#define pMS2B310_INPUTCHANNEL(blockette) ((char *)((uint8_t*)blockette+28)) +#define pMS2B310_RESERVED2(blockette) ((uint8_t*)((uint8_t*)blockette+31)) +#define pMS2B310_REFERENCEAMPLITUDE(blockette) ((uint32_t*)((uint8_t*)blockette+32)) +#define pMS2B310_COUPLING(blockette) ((char*)((uint8_t*)blockette+36)) +#define pMS2B310_ROLLOFF(blockette) ((char*)((uint8_t*)blockette+48)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 320 - pseudo-random calibration + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * year uint16_t 4 + * day uint16_t 6 + * hour uint8_t 8 + * min uint8_t 9 + * sec uint8_t 10 + * unused uint8_t 11 + * fract uint16_t 12 + * reserved1 uint8_t 14 + * flags uint8_t 15 + * duration uint32_t 16 + * PtP amplitude float 20 + * input channel char[3] 24 + * reserved2 uint8_t 27 + * reference amplitude uint32_t 28 + * coupling char[12] 32 + * rolloff char[12] 44 + * noise type char[8] 56 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B320_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B320_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B320_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+4)) +#define pMS2B320_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+6)) +#define pMS2B320_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+8)) +#define pMS2B320_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+9)) +#define pMS2B320_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+10)) +#define pMS2B320_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+11)) +#define pMS2B320_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+12)) +#define pMS2B320_RESERVED1(blockette) ((uint8_t*)((uint8_t*)blockette+14)) +#define pMS2B320_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+15)) +#define pMS2B320_DURATION(blockette) ((uint32_t*)((uint8_t*)blockette+16)) +#define pMS2B320_PTPAMPLITUDE(blockette) ((float*)((uint8_t*)blockette+20)) +#define pMS2B320_INPUTCHANNEL(blockette) ((char *)((uint8_t*)blockette+24)) +#define pMS2B320_RESERVED2(blockette) ((uint8_t*)((uint8_t*)blockette+27)) +#define pMS2B320_REFERENCEAMPLITUDE(blockette) ((uint32_t*)((uint8_t*)blockette+28)) +#define pMS2B320_COUPLING(blockette) ((char*)((uint8_t*)blockette+32)) +#define pMS2B320_ROLLOFF(blockette) ((char*)((uint8_t*)blockette+44)) +#define pMS2B320_NOISETYPE(blockette) ((char*)((uint8_t*)blockette+56)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 390 - generic calibration + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * year uint16_t 4 + * day uint16_t 6 + * hour uint8_t 8 + * min uint8_t 9 + * sec uint8_t 10 + * unused uint8_t 11 + * fract uint16_t 12 + * reserved1 uint8_t 14 + * flags uint8_t 15 + * duration uint32_t 16 + * amplitude float 20 + * input channel char[3] 24 + * reserved2 uint8_t 27 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B390_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B390_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B390_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+4)) +#define pMS2B390_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+6)) +#define pMS2B390_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+8)) +#define pMS2B390_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+9)) +#define pMS2B390_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+10)) +#define pMS2B390_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+11)) +#define pMS2B390_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+12)) +#define pMS2B390_RESERVED1(blockette) ((uint8_t*)((uint8_t*)blockette+14)) +#define pMS2B390_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+15)) +#define pMS2B390_DURATION(blockette) ((uint32_t*)((uint8_t*)blockette+16)) +#define pMS2B390_AMPLITUDE(blockette) ((float*)((uint8_t*)blockette+20)) +#define pMS2B390_INPUTCHANNEL(blockette) ((char *)((uint8_t*)blockette+24)) +#define pMS2B390_RESERVED2(blockette) ((uint8_t*)((uint8_t*)blockette+27)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 395 - calibration abort + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * year uint16_t 4 + * day uint16_t 6 + * hour uint8_t 8 + * min uint8_t 9 + * sec uint8_t 10 + * unused uint8_t 11 + * fract uint16_t 12 + * reserved uint8_t[2] 14 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B395_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B395_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B395_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+4)) +#define pMS2B395_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+6)) +#define pMS2B395_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+8)) +#define pMS2B395_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+9)) +#define pMS2B395_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+10)) +#define pMS2B395_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+11)) +#define pMS2B395_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+12)) +#define pMS2B395_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+14)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 400 - beam + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * azimuth float 4 + * slowness float 8 + * configuration uint16_t 12 + * reserved uint8_t[2] 14 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B400_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B400_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B400_AZIMUTH(blockette) ((float*)((uint8_t*)blockette+4)) +#define pMS2B400_SLOWNESS(blockette) ((float*)((uint8_t*)blockette+8)) +#define pMS2B400_CONFIGURATION(blockette) ((uint16_t*)((uint8_t*)blockette+12)) +#define pMS2B400_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+14)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 405 - beam delay + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * delay values uint16_t[1] 4 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B405_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B405_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B405_DELAYVALUES(blockette) ((uint16_t*)((uint8_t*)blockette+4)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 500 - timing + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * VCO correction float 4 + * year uint16_t 8 + * day uint16_t 10 + * hour uint8_t 12 + * min uint8_t 13 + * sec uint8_t 14 + * unused uint8_t 15 + * fract uint16_t 16 + * microsecond int8_t 18 + * reception quality uint8_t 19 + * exception count uint32_t 20 + * exception type char[16] 24 + * clock model char[32] 40 + * clock status char[128] 72 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B500_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B500_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B500_VCOCORRECTION(blockette) ((float*)((uint8_t*)blockette+4)) +#define pMS2B500_YEAR(blockette) ((uint16_t*)((uint8_t*)blockette+8)) +#define pMS2B500_DAY(blockette) ((uint16_t*)((uint8_t*)blockette+10)) +#define pMS2B500_HOUR(blockette) ((uint8_t*)((uint8_t*)blockette+12)) +#define pMS2B500_MIN(blockette) ((uint8_t*)((uint8_t*)blockette+13)) +#define pMS2B500_SEC(blockette) ((uint8_t*)((uint8_t*)blockette+14)) +#define pMS2B500_UNUSED(blockette) ((uint8_t*)((uint8_t*)blockette+15)) +#define pMS2B500_FSEC(blockette) ((uint16_t*)((uint8_t*)blockette+16)) +#define pMS2B500_MICROSECOND(blockette) ((int8_t*)((uint8_t*)blockette+18)) +#define pMS2B500_RECEPTIONQUALITY(blockette) ((uint8_t*)((uint8_t*)blockette+19)) +#define pMS2B500_EXCEPTIONCOUNT(blockette) ((uint32_t*)((uint8_t*)blockette+20)) +#define pMS2B500_EXCEPTIONTYPE(blockette) ((char*)((uint8_t*)blockette+24)) +#define pMS2B500_CLOCKMODEL(blockette) ((char*)((uint8_t*)blockette+40)) +#define pMS2B500_CLOCKSTATUS(blockette) ((char*)((uint8_t*)blockette+72)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 1000 - data only SEED (miniSEED) + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * encoding uint8_t 4 + * byteorder uint8_t 5 + * reclen uint8_t 6 + * reserved uint8_t 7 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B1000_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B1000_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B1000_ENCODING(blockette) ((uint8_t*)((uint8_t*)blockette+4)) +#define pMS2B1000_BYTEORDER(blockette) ((uint8_t*)((uint8_t*)blockette+5)) +#define pMS2B1000_RECLEN(blockette) ((uint8_t*)((uint8_t*)blockette+6)) +#define pMS2B1000_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+7)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 1001 - data extension + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * timing quality uint8_t 4 + * microsecond int8_t 5 + * reserved uint8_t 6 + * frame count uint8_t 7 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B1001_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B1001_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B1001_TIMINGQUALITY(blockette) ((uint8_t*)((uint8_t*)blockette+4)) +#define pMS2B1001_MICROSECOND(blockette) ((int8_t*)((uint8_t*)blockette+5)) +#define pMS2B1001_RESERVED(blockette) ((uint8_t*)((uint8_t*)blockette+6)) +#define pMS2B1001_FRAMECOUNT(blockette) ((uint8_t*)((uint8_t*)blockette+7)) + +/*************************************************************************** + * miniSEED 2.4 Blockette 2000 - opaque data + * + * FIELD TYPE OFFSET + * type uint16_t 0 + * next offset uint16_t 2 + * length uint16_t 4 + * data offset uint16_t 6 + * recnum uint32_t 8 + * byteorder uint8_t 12 + * flags uint8_t 13 + * numheaders uint8_t 14 + * payload char[1] 15 + * + * Convenience macros for accessing the fields via typed pointers follow: + ***************************************************************************/ +#define pMS2B2000_TYPE(blockette) ((uint16_t*)(blockette)) +#define pMS2B2000_NEXT(blockette) ((uint16_t*)((uint8_t*)blockette+2)) +#define pMS2B2000_LENGTH(blockette) ((uint16_t*)((uint8_t*)blockette+4)) +#define pMS2B2000_DATAOFFSET(blockette) ((uint16_t*)((uint8_t*)blockette+6)) +#define pMS2B2000_RECNUM(blockette) ((uint32_t*)((uint8_t*)blockette+8)) +#define pMS2B2000_BYTEORDER(blockette) ((uint8_t*)((uint8_t*)blockette+12)) +#define pMS2B2000_FLAGS(blockette) ((uint8_t*)((uint8_t*)blockette+13)) +#define pMS2B2000_NUMHEADERS(blockette) ((uint8_t*)((uint8_t*)blockette+14)) +#define pMS2B2000_PAYLOAD(blockette) ((char*)((uint8_t*)blockette+15)) + +/*************************************************************************** + * Simple static inline convenience functions to swap bytes to "host + * order", as determined by the swap flag. + ***************************************************************************/ +static inline int16_t +HO2d (int16_t value, int swapflag) +{ + if (swapflag) + { + ms_gswap2 (&value); + } + return value; +} +static inline uint16_t +HO2u (uint16_t value, int swapflag) +{ + if (swapflag) + { + ms_gswap2 (&value); + } + return value; +} +static inline int32_t +HO4d (int32_t value, int swapflag) +{ + if (swapflag) + { + ms_gswap4 (&value); + } + return value; +} +static inline uint32_t +HO4u (uint32_t value, int swapflag) +{ + if (swapflag) + { + ms_gswap4 (&value); + } + return value; +} +static inline float +HO4f (float value, int swapflag) +{ + if (swapflag) + { + ms_gswap4 (&value); + } + return value; +} +static inline double +HO8f (double value, int swapflag) +{ + if (swapflag) + { + ms_gswap8 (&value); + } + return value; +} + +/* Macro to test for sane year and day values, used primarily to + * determine if byte order swapping is needed for miniSEED 2.x. + * + * Year : between 1900 and 2100 + * Day : between 1 and 366 + * + * This test is non-unique (non-deterministic) for days 1, 256 and 257 + * in the year 2056 because the swapped values are also within range. + * If you are using this in 2056 to determine the byte order of miniSEED 2 + * you have my deepest sympathies. + */ +#define MS_ISVALIDYEARDAY(Y,D) (Y >= 1900 && Y <= 2100 && D >= 1 && D <= 366) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/libs/gempa/caps/CHANGELOG.md b/libs/gempa/caps/CHANGELOG.md index 14d034f..39c77b1 100644 --- a/libs/gempa/caps/CHANGELOG.md +++ b/libs/gempa/caps/CHANGELOG.md @@ -2,149 +2,357 @@ All notable changes to the CAPS client library will be documented in this file. -## 2021-04-21 1.0.0 +## 2026.069 3.2.0 + +### Added + +- Allow to override host operating system information + +## 2026.068 3.1.1 + +### Added + +- Support system load reporting + +## 2026.057 3.1.0 + +### Added + +- Support for CAPS API 7 features + +## 2025.206 3.0.1 + +### Added + +- Stream ID check. Before sending data, the stream ID is checked to ensure + that it meets the server requirements. Specical characters like '?' are + not allowed. + +## 2025.163 3.0.0 + ### Changed -- Set library version to 1.0.0 + +- Make session table item start and end time optional + +## 2024.352 2.7.4 + +### Changed + +- Optimize package encoding + +## 2024.226 2.7.3 + +### Fixed + +- Parse server output address correctly + +## 2024.162 2.7.2 + +### Fixed + +- Do not crash when the journal file does not follow the format + +## 2024.162 2.7.1 + +### Changed + +- Redirect log messages to SeisComP log by default + +## 2024.151 2.7.0 + +### Added + +- Add plugin instance pointer to log messages + +## 2023.184 2.6.0 + +### Added + +- More convenience push functions + +## 2023.184 2.5.0 + +### Added + +- Direct API access to last acknowledged packet end time + +## 2023.107 2.4.0 + +### Added + +- PluginApplication class logs CAPS connection settings + +### Changed + +- Reworked connection handling to perfom better in poor network environments +- Removed unnecessary debug output + +## 2022.284 2.3.1 + +### Changed + +- Minor changes + +## 2022-04-20 2.3.0 + +### Added + +- CAPS agent support + +## 2022-04-20 2.2.0 + +### Added + +- API function to access internal packet buffer that keeps packets until + they have been acknowledged by CAPS + +## 2021-09-01 2.1.0 + +### Added + +- Extended plugin class by the possibility to dump **outgoing** RAW and MSEED + packets to stdout. RAW data ouput is in SLIST format and MSEED packets are + dumped as binary MSEED. Applications derived from the plugin application class + can enable this feature with the command line option ``--dump-packets``. + +## 2021-06-21 2.0.3 + +### Fixed + +- CAPS Python encoder factory ownership + +## 2021-06-15 2.0.2 + +### Fixed + +- Broken uncompressed MSEED support + +## 2021-05-10 2.0.1 + +### Fixed + +- Unit tests that work on 64-bit systems only + +## 2021-05-10 2.0.0 + +### Added + +- Log status responses from server in plugin application +- SSL and authentication support for plugin application. + With this version the data output URL can be set with the + config option ``output.address``. The formal definition of + the field is: [[caps|capss]://][user:pass@]host[:port] e.g. + + ```script + output.address = capss://caps:caps@localhost:18003 + ``` + + The new output.address parameter superseds the output.host and + output.port parameter of previous versions and takes precedence. + The old parameters are kept for compatibility reasons but are + marked as deprecated. + +## 2021-04-21 1.0.0 + +### Changed + +- Set library version to 1.0.0 ## 2021-02-16 + ### Added -- Allow to set maximum allowed future end time via plugin API. In additon - add commandline and config support to the plugin application class e.g. - ``` - output.maxFutureEndTime = 120 - ``` - By default the option is set to 120 seconds. + +- Allow to set maximum allowed future end time via plugin API. In additon + add commandline and config support to the plugin application class e.g. + + ```script + output.maxFutureEndTime = 120 + ``` + + By default the option is set to 120 seconds. ## 2020-12-14 + ### Added -- Support to set miniSEED record length via API + +- Support to set miniSEED record length via API ## 2020-09-22 + ### Changed -- Use last sample time as reference time for maximum future time check. Before - we used the packet end time as reference time but this makes no sense in case - of low sampled data. -- So far we used the current time as start time if journal entries exceeded the - maximum allowed future time. With this release we use the journal time no - matter what its value but display a warning when the journal time is more than - one day in the future. + +- Use last sample time as reference time for maximum future time check. Before + we used the packet end time as reference time but this makes no sense in case + of low sampled data. +- So far we used the current time as start time if journal entries exceeded the + maximum allowed future time. With this release we use the journal time no + matter what its value but display a warning when the journal time is more than + one day in the future. ## 2020-06-22 + ### Added -- Python3 byte array support to any record push + +- Python3 byte array support to any record push ## 2020-02-21 + ### Changed -- Increase default timeout for acknowledgement messages from 5s to 60s + +- Increase default timeout for acknowledgement messages from 5s to 60s ## 2019-09-24 + ### Fixed -- Fix high load if packets could not be sent to CAPS. In that case the plugin - automatically reconnects after some amount of time. If triggered under certain - circumstances this delay was not in effect and caused unnecessarily high - amount of connection attempts. Under some circumstances the plugin could have - crashed due to a stack overflow. -- The quit method has now the semantics to continue pushing packets as long - as the connection is established and only abort in case of an error without - attempting a reconnect. + +- Fix high load if packets could not be sent to CAPS. In that case the plugin + automatically reconnects after some amount of time. If triggered under certain + circumstances this delay was not in effect and caused unnecessarily high + amount of connection attempts. Under some circumstances the plugin could have + crashed due to a stack overflow. +- The quit method has now the semantics to continue pushing packets as long + as the connection is established and only abort in case of an error without + attempting a reconnect. ## 2019-09-20 + ### Fixed -- Fix error string if not all data could be sent to the server + +- Fix error string if not all data could be sent to the server ## 2019-08-19 + ### Changed -- Discard packets whose end time is more than 120 seconds before the system time. + +- Discard packets whose end time is more than 120 seconds before the system time. ## 2019-08-06 + ### Added -- new config option ``output.addr`` + +- new config option ``output.addr`` ### Fixed -- ambiguous command line option ``-h``. With this version of the library the -host and port can be set via the command line option ``--addr``. -- wrong config option parsing + +- ambiguous command line option ``-h``. With this version of the library the + host and port can be set via the command line option ``--addr``. +- wrong config option parsing ## 2019-08-05 + ### Fixed -- seg fault in date time parser + +- seg fault in date time parser ## 2019-08-02 + ### Fixed -- Hanging TCP connections. In case of the remote side does not shutdown cleanly -the plugin did not notice that the connection is no longer available. With this -version the plugin reconnects to the server when the TCP send buffer is full and -tries to send all not acknowledged packets again. -- Do not discard packets if the packet buffer is full. Instead we block until -the server acknowledges some packets. + +- Hanging TCP connections. In case of the remote side does not shutdown cleanly + the plugin did not notice that the connection is no longer available. With this + version the plugin reconnects to the server when the TCP send buffer is full + and tries to send all not acknowledged packets again. +- Do not discard packets if the packet buffer is full. Instead we block until + the server acknowledges some packets. ### Changed -- The plugin application class checks whether the configured buffer size is -below the minimum. + +- The plugin application class checks whether the configured buffer size is + below the minimum. ## 2019-07-05 + ### Changed -- Ignore journal entries where the timestamp is more than 10 seconds - before the system time. + +- Ignore journal entries where the timestamp is more than 10 seconds + before the system time. ## 2018-12-19 + ### Fixed -- Read journal from file in plugin application. + +- Read journal from file in plugin application. ## 2018-12-18 + ### Fixed -- Do not reconnect if the plugin buffer is full. Instead of we try to read -acknowledgements from the CAPS server until the plugin buffer is below the -threshold. + +- Do not reconnect if the plugin buffer is full. Instead of we try to read + acknowledgements from the CAPS server until the plugin buffer is below the + threshold. ## 2018-12-17 + ### Added -- Support to retrieve status information e.g. the number of buffered bytes from - plugin. + +- Support to retrieve status information e.g. the number of buffered bytes from + plugin. ## 2018-09-06 + ### Changed -- Enable more verbose logging for MSEED packets + +- Enable more verbose logging for MSEED packets ## 2018-07-25 + ### Fixed -- unset variable of the raw data record -- trim function will return false in case of an unknown datatype + +- unset variable of the raw data record +- trim function will return false in case of an unknown datatype ## 2018-05-30 + ### Fixed -- Fixed unexpected closed SSL connections + +- Fixed unexpected closed SSL connections ## 2018-06-05 + ### Fixed -- Fix RawDataRecord::setHeader + +- Fix RawDataRecord::setHeader ## 2018-05-16 + ### Fixed -- RAW data end time calculation + +- RAW data end time calculation ## 2018-03-19 -### Added -- SSL support + +### Fixed + +- SSL support ## 2017-11-20 + ### Added -- float and double support for Steim encoders. All values will be converted implicitly -to int 32 values + +- Float and double support for Steim encoders. All values will be converted + implicitly to int 32 values. ## 2017-11-08 + ### Added -- timing quality parameter to push call. By default the timing quality is set to -1. + +- Timing quality parameter to push call. By default the timing quality is set + to -1. ## 2017-11-07 + ### Fixed -- do not flush encoders after reconnect + +- Do not flush encoders after reconnect ## 2017-10-26 -### Added -- SSL support +### Added + +- SSL support ## 2017-10-24 + ### Fixed -- packet synchronization error after reconnect + +- Packet synchronization error after reconnect diff --git a/libs/gempa/caps/anypacket.cpp b/libs/gempa/caps/anypacket.cpp index aabf1b8..03d7078 100644 --- a/libs/gempa/caps/anypacket.cpp +++ b/libs/gempa/caps/anypacket.cpp @@ -84,6 +84,11 @@ void AnyDataRecord::setSamplingFrequency(uint16_t numerator, uint16_t denominato } +DataRecord *AnyDataRecord::clone() const { + return new AnyDataRecord(*this); +} + + const char *AnyDataRecord::formatName() const { return "ANY"; } diff --git a/libs/gempa/caps/anypacket.h b/libs/gempa/caps/anypacket.h index cbe180a..a7749c7 100644 --- a/libs/gempa/caps/anypacket.h +++ b/libs/gempa/caps/anypacket.h @@ -20,11 +20,11 @@ #include #include -#include namespace Gempa { namespace CAPS { + class AnyDataRecord : public DataRecord { public: typedef std::vector Buffer; @@ -74,37 +74,37 @@ class AnyDataRecord : public DataRecord { bool setType(const char *type); const char *type() const; - virtual const char *formatName() const; + DataRecord *clone() const override; + const char *formatName() const override; - virtual bool readMetaData(std::streambuf &buf, int size, - Header &header, - Time &startTime, - Time &endTime); + bool readMetaData(std::streambuf &buf, int size, + Header &header, + Time &startTime, + Time &endTime) override; - virtual const Header *header() const; - virtual Time startTime() const; - virtual Time endTime() const; + const Header *header() const override; + Time startTime() const override; + Time endTime() const override; - virtual bool canTrim() const; - virtual bool canMerge() const; + bool canTrim() const override; + bool canMerge() const override; - virtual bool trim(const Time &start, - const Time &end) const; + bool trim(const Time &start, const Time &end) const override; - virtual size_t dataSize(bool withHeader) const; + size_t dataSize(bool withHeader) const override; - virtual ReadStatus get(std::streambuf &buf, int size, - const Time &start = Time(), - const Time &end = Time(), - int maxSize = -1); + ReadStatus get(std::streambuf &buf, int size, + const Time &start = Time(), + const Time &end = Time(), + int maxSize = -1) override; - virtual bool put(std::streambuf &buf, bool withHeader) const; + bool put(std::streambuf &buf, bool withHeader) const override; /** * @brief Returns the packet type * @return The packet type */ - PacketType packetType() const { return ANYPacket; } + PacketType packetType() const override { return ANYPacket; } /** * @brief Sets the start time of the record @@ -129,21 +129,22 @@ class AnyDataRecord : public DataRecord { * @brief Returns the data vector to be filled by the caller * @return The pointer to the internal buffer */ - Buffer *data() { return &_data; } + Buffer *data() override { return &_data; } /** * @brief Initializes the internal data vector from the given buffer * @param The buffer to read the data from * @param The buffer size */ - virtual void setData(char *data, size_t size); + void setData(char *data, size_t size); + protected: - AnyHeader _header; - Buffer _data; + AnyHeader _header; + Buffer _data; - Time _startTime; - Time _endTime; + Time _startTime; + Time _endTime; }; diff --git a/libs/gempa/caps/application.h b/libs/gempa/caps/application.h index c8d64bf..d046cbf 100644 --- a/libs/gempa/caps/application.h +++ b/libs/gempa/caps/application.h @@ -16,11 +16,14 @@ #ifndef GEMPA_CAPS_APPLICATION_H #define GEMPA_CAPS_APPLICATION_H + #include + namespace Gempa { namespace CAPS { + class SC_GEMPA_CAPS_API Application { public: Application(int argc, char **argv); @@ -99,6 +102,7 @@ class SC_GEMPA_CAPS_API Application { static Application *_app; }; + } } diff --git a/libs/gempa/caps/connection.h b/libs/gempa/caps/connection.h index e7dcf6f..dc26649 100644 --- a/libs/gempa/caps/connection.h +++ b/libs/gempa/caps/connection.h @@ -16,6 +16,7 @@ #ifndef GEMPA_CAPS_CONNECTION_H #define GEMPA_CAPS_CONNECTION_H + #include #include #include @@ -27,12 +28,15 @@ //#include #include + namespace Gempa { namespace CAPS { -class SessionTableItem; + +struct SessionTableItem; class Time; + class Connection { public: //! ConnectionStates: @@ -171,10 +175,12 @@ class Connection { bool _ssl; }; -typedef boost::shared_ptr ConnectionPtr; + +using ConnectionPtr = boost::shared_ptr; } } + #endif diff --git a/libs/gempa/caps/datetime.cpp b/libs/gempa/caps/datetime.cpp index bf84c5a..de37d9d 100644 --- a/libs/gempa/caps/datetime.cpp +++ b/libs/gempa/caps/datetime.cpp @@ -18,9 +18,10 @@ #include #include #include -#include -#include -#include +#include +#include +#include +#include #ifdef WIN32 #include @@ -168,6 +169,26 @@ inline void normalize(T &sec, U &usec) { } } +const char *timeFormats[] = { + "%FT%T.%fZ", // YYYY-MM-DDThh:mm:ss.ssssssZ + "%FT%T.%f", // YYYY-MM-DDThh:mm:ss.ssssss + "%FT%TZ", // YYYY-MM-DDThh:mm:ssZ + "%FT%T", // YYYY-MM-DDThh:mm:ss + "%FT%R", // YYYY-MM-DDThh:mm + "%FT%H", // YYYY-MM-DDThh + "%Y-%jT%T.%f", // YYYY-DDDThh:mm:ss.ssssss + "%Y-%jT%T", // YYYY-DDDThh:mm:ss + "%Y-%jT%R", // YYYY-DDDThh:mm + "%Y-%jT%H", // YYYY-DDDThh + "%F %T.%f", // YYYY-MM-DD hh:mm:ss.ssssss + "%F %T", // YYYY-MM-DD hh:mm:ss + "%F %R", // YYYY-MM-DD hh:mm + "%F %H", // YYYY-MM-DD hh + "%F", // YYYY-MM-DD + "%Y-%j", // YYYY-DDD + "%Y", // YYYY +}; + } // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @@ -342,6 +363,17 @@ TimeSpan& TimeSpan::operator=(long t) { +// >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +TimeSpan& TimeSpan::operator=(const TimeSpan& t) { + _timeval = t._timeval; + + return *this; +} +// <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + + + // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> TimeSpan& TimeSpan::operator=(double t) { if( t > (double)0x7fffffff || t < -(double)0x80000000 ) @@ -801,17 +833,10 @@ Time& Time::localtime() { // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -Time& Time::gmt() { - gettimeofday(&_timeval, NULL); - time_t secs = (time_t)_timeval.tv_sec; - struct tm _tm; -#ifndef WIN32 - _timeval.tv_sec = (long)mktime(::localtime_r(&secs, &_tm)); -#else - // We use the native localtime function of windows, which is thread safe. (But it's not reentrant) - _timeval.tv_sec = (long)timegm(::localtime(&secs)); -#endif - +Time &Time::gmt() { + auto us = chrono::duration_cast(chrono::high_resolution_clock::now().time_since_epoch()).count(); + _timeval.tv_sec = us / 1000000; + _timeval.tv_usec = us % 1000000; return *this; } // <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @@ -1055,9 +1080,50 @@ bool Time::fromString(const char* str, const char* fmt) { +// >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +bool Time::fromString(const char* str) { + for ( size_t i = 0; i < sizeof(timeFormats) / sizeof(const char*); ++i ) { + if ( fromString(str, timeFormats[i]) ) { + return true; + } + } + return false; +} +// <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + + +// >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +bool Time::fromString(const std::string& str) { + return fromString(str.c_str()); +} +// <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + + // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Time Time::FromString(const char* str, const char* fmt) { Time t; t.fromString(str, fmt); return t; } +// <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + + +// >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +Time Time::FromString(const char* str) { + Time t; + t.fromString(str); + return t; +} +// <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + + +// >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +Time Time::FromString(const std::string& str) { + return FromString(str.c_str()); +} +// <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + diff --git a/libs/gempa/caps/datetime.h b/libs/gempa/caps/datetime.h index ef8b4e0..e42f0e6 100644 --- a/libs/gempa/caps/datetime.h +++ b/libs/gempa/caps/datetime.h @@ -16,6 +16,7 @@ #ifndef GEMPA_CAPS_DATETIME_H #define GEMPA_CAPS_DATETIME_H + #ifdef WIN32 #include #else @@ -24,8 +25,10 @@ #include + struct tm; + namespace Gempa { namespace CAPS { @@ -64,6 +67,7 @@ class TimeSpan { //! Assignment TimeSpan& operator=(long t); TimeSpan& operator=(double t); + TimeSpan& operator=(const TimeSpan& t); //! Arithmetic TimeSpan operator+(const TimeSpan&) const; @@ -205,24 +209,25 @@ class Time : public TimeSpan { //! Returns whether the date is valid or not bool valid() const; - /** Converts the time to string using format fmt. - @param fmt The format string can contain any specifiers - as allowed for strftime. Additional the '%f' - specifier is replaced by the fraction of the seconds. - Example: - toString("%FT%T.%fZ") = "1970-01-01T00:00:00.0000Z" - @return A formatted string + /** + * @brief Converts the time to string using format fmt. + * @param fmt The format string can contain any specifiers + * as allowed for strftime. Additional the '%f' + * specifier is replaced by the fraction of the seconds. + * Example: + * toString("%FT%T.%fZ") = "1970-01-01T00:00:00.0000Z" + * @return A formatted string */ std::string toString(const char* fmt) const; /** - * Converts the time to a string using the ISO time description + * @brief Converts the time to a string using the ISO time description * @return A formatted string */ std::string iso() const; /** - * Converts a string into a time representation. + * @brief Converts a string into a time representation. * @param str The string representation of the time * @param fmt The format string containing the conversion * specification (-> toString) @@ -230,11 +235,37 @@ class Time : public TimeSpan { */ bool fromString(const char* str, const char* fmt); + /** + * @brief Converts a string into a time representation trying a common + * set of date formats. + * @param str The string representation of the time. + * @return The conversion result + */ + bool fromString(const char* str); + + /** + * @brief Convenience method for fromString(const char*). + */ + bool fromString(const std::string& str); + /** * Static method to create a time value from a string. * The parameters are the same as in Time::fromString. */ static Time FromString(const char* str, const char* fmt); + + /** + * @brief Static methods that converts a string into a time + * representation trying a common set of date formats. + * @param str The string representation of the time. + * @return The conversion result + */ + static Time FromString(const char* str); + + /** + * @brief Convenience method for FromString(const char*). + */ + static Time FromString(const std::string& str); }; diff --git a/libs/gempa/caps/encoderfactory.h b/libs/gempa/caps/encoderfactory.h index 1c72b03..1480af1 100644 --- a/libs/gempa/caps/encoderfactory.h +++ b/libs/gempa/caps/encoderfactory.h @@ -12,16 +12,19 @@ * from gempa GmbH. * ***************************************************************************/ + #ifndef GEMPA_CAPS_ENCODERFACTORY_H #define GEMPA_CAPS_ENCODERFACTORY_H -#include "mseed/encoder.h" +#include "mseed/encoder.h" #include + namespace Gempa { namespace CAPS { + /** * @brief Abstract base class of the encoder factory. Each * derived class must implement the create and the @@ -158,7 +161,9 @@ class Steim2EncoderFactory : public SteimEncoderFactory { int samplingFrequencyDenominator); }; + } } + #endif diff --git a/libs/gempa/caps/endianess.h b/libs/gempa/caps/endianess.h index 75a67e7..279c420 100644 --- a/libs/gempa/caps/endianess.h +++ b/libs/gempa/caps/endianess.h @@ -16,6 +16,7 @@ #ifndef GEMPA_CAPS_ENDIANESS_H #define GEMPA_CAPS_ENDIANESS_H + #include #include #include @@ -25,6 +26,7 @@ namespace Gempa { namespace CAPS { namespace Endianess { + template inline const T1* lvalue(const T2 &value) { return reinterpret_cast(&value); } @@ -203,8 +205,10 @@ struct Writer { bool good; }; + } } } + #endif diff --git a/libs/gempa/caps/metapacket.cpp b/libs/gempa/caps/metapacket.cpp index 6a181e8..1b18d29 100644 --- a/libs/gempa/caps/metapacket.cpp +++ b/libs/gempa/caps/metapacket.cpp @@ -42,6 +42,10 @@ MetaDataRecord::Buffer *MetaDataRecord::data() { return NULL; } +DataRecord *MetaDataRecord::clone() const { + return new MetaDataRecord(*this); +} + const char *MetaDataRecord::formatName() const { return "META"; } diff --git a/libs/gempa/caps/metapacket.h b/libs/gempa/caps/metapacket.h index 46abb61..676a52e 100644 --- a/libs/gempa/caps/metapacket.h +++ b/libs/gempa/caps/metapacket.h @@ -16,19 +16,18 @@ #ifndef GEMPA_CAPS_METAPACKET_H #define GEMPA_CAPS_METAPACKET_H + #include #include -#include -#include -#include - +#include namespace Gempa { namespace CAPS { -struct SC_GEMPA_CAPS_API MetaResponseHeader { + + struct SC_GEMPA_CAPS_API MetaResponseHeader { struct Time { int64_t seconds; int32_t microSeconds; @@ -95,15 +94,16 @@ class MetaDataRecord : public DataRecord { //! Returns the data vector to be filled by the caller Buffer *data(); - virtual const char *formatName() const; + DataRecord *clone() const override; + const char *formatName() const override; - virtual bool readMetaData(std::streambuf &buf, int size, - Header &header, - Time &startTime, Time &endTime) { + bool readMetaData(std::streambuf &buf, int size, + Header &header, + Time &startTime, Time &endTime) override { return false; } - virtual const Header *header() const; + const Header *header() const override; //! Sets a custom header and updates all internal structures //! based on the current data. If the data has changed, make @@ -111,23 +111,24 @@ class MetaDataRecord : public DataRecord { //! is set get will not read the header information from stream void setHeader(const MetaHeader &header); - virtual Time startTime() const; - virtual Time endTime() const; + Time startTime() const override; + Time endTime() const override; - virtual bool canTrim() const { return false; } - virtual bool canMerge() const { return false; } + bool canTrim() const override { return false; } + bool canMerge() const override { return false; } - virtual bool trim(const Time &start, const Time &end) const { return false; } + bool trim(const Time &start, const Time &end) const override { return false; } - virtual size_t dataSize(bool withHeader) const; + size_t dataSize(bool withHeader) const override; - virtual ReadStatus get(std::streambuf &buf, int size, - const Time &start, const Time &end, - int maxBytes); + ReadStatus get(std::streambuf &buf, int size, + const Time &start, const Time &end, + int maxBytes) override; - virtual bool put(std::streambuf &buf, bool withHeader) const { return false; } + bool put(std::streambuf &buf, bool withHeader) const override { return false; } + + PacketType packetType() const override { return MetaDataPacket; } - PacketType packetType() const { return MetaDataPacket; } private: MetaHeader _header; @@ -136,6 +137,7 @@ class MetaDataRecord : public DataRecord { mutable Time _endTime; }; + } } diff --git a/libs/gempa/caps/mseed/encoder.h b/libs/gempa/caps/mseed/encoder.h index 26c4322..f56673c 100644 --- a/libs/gempa/caps/mseed/encoder.h +++ b/libs/gempa/caps/mseed/encoder.h @@ -35,8 +35,11 @@ namespace CAPS { class Encoder { public: - Encoder(int freqn, int freqd) : _clk(freqn, freqd), - _sampleCount(0), _timingQuality(-1) {} + Encoder(int freqn, int freqd) + : _clk(freqn, freqd) + , _sampleCount(0), _timingQuality(-1) + , _context{nullptr} {} + virtual ~Encoder() {} virtual void push(void *sample) = 0; @@ -49,13 +52,18 @@ class Encoder { void setStartTime(const Time &time) { _clk.syncTime(time); } const Time currentTime() const { return _clk.getTime(0); } - int timingQuality() { return _timingQuality; } void setTimingQuality(int quality) { _timingQuality = quality; } + int timingQuality() { return _timingQuality; } + + void setContext(void *context) { _context = context; } PacketPtr pop() { - if ( _packetQueue.empty() ) return PacketPtr(); + if ( _packetQueue.empty() ) { + return PacketPtr(); + } PacketPtr rec = _packetQueue.front(); + rec->context = _context; _packetQueue.pop_front(); return rec; } @@ -66,6 +74,7 @@ class Encoder { int _sampleCount; PacketQueue _packetQueue; int _timingQuality; + void *_context; }; } diff --git a/libs/gempa/caps/mseed/mseed.cpp b/libs/gempa/caps/mseed/mseed.cpp index a3d2113..4ae3f7c 100644 --- a/libs/gempa/caps/mseed/mseed.cpp +++ b/libs/gempa/caps/mseed/mseed.cpp @@ -143,10 +143,9 @@ MSEEDDataRecord *MSEEDFormat::getBuffer(const Time &it, int usec_correction, void MSEEDFormat::updateBuffer(MSEEDDataRecord *rec, int samples, int frames) { sl_fsdh_s* fsdh = (sl_fsdh_s *)rec->data()->data(); - char temp[7]; + char temp[6] = { 0, 0, 0, 0, 0, 0 }; - sprintf(temp, "%06d", (int)0); - memcpy(fsdh->sequence_number,temp,6); + memcpy(fsdh->sequence_number, temp, 6); fsdh->dhq_indicator = 'D'; fsdh->num_samples = htons(samples); @@ -157,6 +156,8 @@ void MSEEDFormat::updateBuffer(MSEEDDataRecord *rec, int samples, int frames) { sizeof(sl_fsdh_s) + sizeof(sl_blkt_1000_s)); blkt_1001->frame_cnt = frames; } + + rec->unpackHeader(); } diff --git a/libs/gempa/caps/mseed/steim1.ipp b/libs/gempa/caps/mseed/steim1.ipp index e126705..26163f8 100644 --- a/libs/gempa/caps/mseed/steim1.ipp +++ b/libs/gempa/caps/mseed/steim1.ipp @@ -36,12 +36,12 @@ Steim1Encoder::~Steim1Encoder() { template void Steim1Encoder::updateSpw(int bp) { - int spw1 = 4; + int spw = 4; assert(bp < 4); - if ( _buf[bp] < -32768 || _buf[bp] > 32767 ) spw1 = 1; - else if ( _buf[bp] < -128 || _buf[bp] > 127 ) spw1 = 2; - if ( spw1 < _spw ) _spw = spw1; + if ( _buf[bp] < -32768 || _buf[bp] > 32767 ) spw = 1; + else if ( _buf[bp] < -128 || _buf[bp] > 127 ) spw = 2; + if ( spw < _spw ) _spw = spw; } template diff --git a/libs/gempa/caps/mseed/steim2.ipp b/libs/gempa/caps/mseed/steim2.ipp index b685d11..47015ea 100644 --- a/libs/gempa/caps/mseed/steim2.ipp +++ b/libs/gempa/caps/mseed/steim2.ipp @@ -38,36 +38,36 @@ template Steim2Encoder::~Steim2Encoder() { } template void Steim2Encoder::updateSpw(int bp) { - assert(_bp < 7); + assert(bp < 7); - if ( _buf[_bp] < -536870912 ) { + if ( _buf[bp] < -536870912 ) { CAPS_WARNING("%s.%s.%s.%s: value %d is too large for Steim2 encoding", _format->networkCode.c_str(), _format->stationCode.c_str(), _format->locationCode.c_str(), _format->channelCode.c_str(), - _buf[_bp]); - _buf[_bp] = -536870912; + _buf[bp]); + _buf[bp] = -536870912; _spw = 1; return; } - if ( _buf[_bp] > 536870911 ) { + if ( _buf[bp] > 536870911 ) { CAPS_WARNING("%s.%s.%s.%s: value %d is too large for Steim2 encoding", _format->networkCode.c_str(), _format->stationCode.c_str(), _format->locationCode.c_str(), _format->channelCode.c_str(), - _buf[_bp]); - _buf[_bp] = 536870911; + _buf[bp]); + _buf[bp] = 536870911; _spw = 1; return; } - int spw1 = 7; - if ( _buf[_bp] < -16384 || _buf[_bp] > 16383 ) spw1 = 1; - else if ( _buf[_bp] < -512 || _buf[_bp] > 511 ) spw1 = 2; - else if ( _buf[_bp] < -128 || _buf[_bp] > 127 ) spw1 = 3; - else if ( _buf[_bp] < -32 || _buf[_bp] > 31 ) spw1 = 4; - else if ( _buf[_bp] < -16 || _buf[_bp] > 15 ) spw1 = 5; - else if ( _buf[_bp] < -8 || _buf[_bp] > 7 ) spw1 = 6; - if ( spw1 < _spw ) _spw = spw1; + int spw = 7; + if ( _buf[bp] < -16384 || _buf[bp] > 16383 ) spw = 1; + else if ( _buf[bp] < -512 || _buf[bp] > 511 ) spw = 2; + else if ( _buf[bp] < -128 || _buf[bp] > 127 ) spw = 3; + else if ( _buf[bp] < -32 || _buf[bp] > 31 ) spw = 4; + else if ( _buf[bp] < -16 || _buf[bp] > 15 ) spw = 5; + else if ( _buf[bp] < -8 || _buf[bp] > 7 ) spw = 6; + if ( spw < _spw ) _spw = spw; } template void Steim2Encoder::store(int32_t value) { diff --git a/libs/gempa/caps/mseedpacket.cpp b/libs/gempa/caps/mseedpacket.cpp index 571553d..25426c7 100644 --- a/libs/gempa/caps/mseedpacket.cpp +++ b/libs/gempa/caps/mseedpacket.cpp @@ -13,16 +13,24 @@ ***************************************************************************/ +#include #include #include #include +#include +#include #include +#include #include #include -#include -namespace { +// Force local include +#include "./3rd-party/libmseed/libmseed.h" +#include "./3rd-party/libmseed/mseedformat.h" + + +#define MS_ISVALIDYEARDAY(Y,D) (Y >= 1900 && Y <= 2100 && D >= 1 && D <= 366) #define LOG_SID(FUNC, format,...)\ do {\ @@ -30,43 +38,151 @@ do {\ char s[6];\ char c[6];\ char l[6];\ - ms_strncpclean(n, head.network, 2);\ - ms_strncpclean(s, head.station, 5);\ - ms_strncpclean(l, head.location, 2);\ - ms_strncpclean(c, head.channel, 3);\ + ms_strncpclean(n, pMS2FSDH_NETWORK(head), 2);\ + ms_strncpclean(s, pMS2FSDH_STATION(head), 5);\ + ms_strncpclean(l, pMS2FSDH_LOCATION(head), 2);\ + ms_strncpclean(c, pMS2FSDH_CHANNEL(head), 3);\ FUNC("[%s.%s.%s.%s] " format, n, s, l, c, ##__VA_ARGS__);\ } while(0) -} namespace Gempa { namespace CAPS { +namespace { + + +uint16_t ms2_blktlen(uint16_t blkttype, const char *blkt, int8_t swapflag) { + uint16_t blktlen = 0; + + switch ( blkttype ) { + case 100: // Sample Rate + blktlen = 12; + break; + case 200: // Generic Event Detection + blktlen = 28; + break; + case 201: // Murdock Event Detection + blktlen = 36; + break; + case 300: // Step Calibration + blktlen = 32; + break; + case 310: // Sine Calibration + blktlen = 32; + break; + case 320: // Pseudo-random Calibration + blktlen = 28; + break; + case 390: // Generic Calibration + blktlen = 28; + break; + case 395: // Calibration Abort + blktlen = 16; + break; + case 400: // Beam + blktlen = 16; + break; + case 500: // Timing + blktlen = 8; + break; + case 1000: // Data Only SEED + blktlen = 8; + break; + case 1001: // Data Extension + blktlen = 8; + break; + case 2000: // Opaque Data + // First 2-byte field after the blockette header is the length + if ( blkt ) { + blktlen = *reinterpret_cast(blkt + 4); + if ( swapflag ) { + ms_gswap2(&blktlen); + } + } + break; + } + + return blktlen; +} + + +bool double2frac(uint16_t &numerator, uint16_t &denominator, double value) { + using namespace std; + + // Check numeric limits + if ( value > numeric_limits::max() ) { + numerator = numeric_limits::max(); + denominator = 1; + return false; + } + + if ( value < numeric_limits::min() ) { + numerator = numeric_limits::min(); + denominator = 1; + return false; + } + + // Operate on positive numbers + int sign = 1; + if ( value < 0 ) { + sign = -1; + value = -value; + } + + // Calculatate the largest possible power of 10 giving numeric integer + // limits and the current input number + static auto max_exp = floor(log10(numeric_limits::max())) - 1.0; + auto exp = max_exp; + if ( value >= 10 ) { + exp -= floor(log10(value)); + } + + // Expand input number with power of 10 + denominator = static_cast(pow(10, exp)); + numerator = static_cast(round(value * denominator)); + + // Simplify the fraction by calculating the greatest common divisor + int gcd_ = gcd(numerator, denominator); + + numerator = sign * numerator / gcd_; + denominator = denominator / gcd_; + return true; +} + + +} + + MSEEDDataRecord::MSEEDDataRecord() {} + +DataRecord *MSEEDDataRecord::clone() const { + return new MSEEDDataRecord(*this); +} + + const char *MSEEDDataRecord::formatName() const { return "MSEED"; } bool MSEEDDataRecord::readMetaData(std::streambuf &buf, int size, - Header &header, - Time &startTime, - Time &endTime) { - fsdh_s head; + Header &header, Time &startTime, Time &endTime) { + char head[48]; if ( size <= 0 ) { CAPS_WARNING("read metadata: invalid size of record: %d", size); return false; } - if ( size < MINRECLEN || size > MAXRECLEN ) { + if ( (size < MINRECLEN) || (size > MAXRECLEN) ) { CAPS_WARNING("read metadata: invalid MSEED record size: %d", size); return false; } - // Read first 32 byte + // Read first 40 byte size_t read = buf.sgetn((char*)&head, sizeof(head)); if ( read < sizeof(head) ) { CAPS_WARNING("read metadata: input buffer underflow: only %d/%d bytes read", @@ -74,172 +190,249 @@ bool MSEEDDataRecord::readMetaData(std::streambuf &buf, int size, return false; } - if ( !MS_ISVALIDHEADER(((char*)&head)) ) { + if ( MS2_ISVALIDHEADER(((char*)&head)) ) { + int headerswapflag = 0; + + // Swap byte order? + if ( !MS_ISVALIDYEARDAY(*pMS2FSDH_YEAR(head), *pMS2FSDH_DAY(head)) ) { + headerswapflag = 1; + + ms_gswap2(pMS2FSDH_YEAR(head)); + ms_gswap2(pMS2FSDH_DAY(head)); + ms_gswap2(pMS2FSDH_FSEC(head)); + + ms_gswap2(pMS2FSDH_NUMSAMPLES(head)); + ms_gswap2(pMS2FSDH_SAMPLERATEFACT(head)); + ms_gswap2(pMS2FSDH_SAMPLERATEMULT(head)); + ms_gswap4(pMS2FSDH_TIMECORRECT(head)); + ms_gswap2(pMS2FSDH_DATAOFFSET(head)); + ms_gswap2(pMS2FSDH_BLOCKETTEOFFSET(head)); + } + + auto year = *pMS2FSDH_YEAR(head); + auto yday = *pMS2FSDH_DAY(head); + auto hours = *pMS2FSDH_HOUR(head); + auto minutes = *pMS2FSDH_MIN(head); + auto seconds = *pMS2FSDH_SEC(head); + auto fsec = *pMS2FSDH_FSEC(head) * 100; + + header.dataType = DT_Unknown; + + startTime = Time::FromYearDay(year, yday); + startTime += TimeSpan(hours * 3600 + minutes * 60 + seconds, fsec); + + header.quality.ID = 0; + header.quality.str[0] = *pMS2FSDH_DATAQUALITY(head); + + if ( *pMS2FSDH_TIMECORRECT(head) != 0 && !(*pMS2FSDH_ACTFLAGS(head) & 0x02) ) { + startTime += TimeSpan(0, *pMS2FSDH_TIMECORRECT(head) * 100); + } + + // Parse blockettes + uint32_t blkt_offset = *pMS2FSDH_BLOCKETTEOFFSET(head); + uint32_t blkt_length; + uint16_t blkt_type; + uint16_t next_blkt; + + if ( blkt_offset < sizeof(head) ) { + LOG_SID(CAPS_DEBUG, "%s", "read metadata: blockette " + "offset points into header"); + return false; + } + + uint32_t coffs = 0; + + while ( (blkt_offset != 0) && ((int)blkt_offset < size) && + (blkt_offset < static_cast(size)) ) { + char bhead[6]; + int seek_ofs = blkt_offset - sizeof(head) - coffs; + buf.pubseekoff(seek_ofs, std::ios_base::cur, std::ios_base::in); + coffs += seek_ofs; + if ( buf.sgetn(bhead, 6) != 6 ) { + LOG_SID(CAPS_DEBUG, "%s", "read metadata: " + "failed to read blockette header"); + break; + } + + coffs += 6; + + memcpy(&blkt_type, bhead, 2); + memcpy(&next_blkt, bhead + 2, 2); + + if ( headerswapflag ) { + ms_gswap2(&blkt_type); + ms_gswap2(&next_blkt); + } + + blkt_length = ms2_blktlen(blkt_type, bhead, headerswapflag); + + if ( blkt_length == 0 ) { + LOG_SID(CAPS_DEBUG, "read metadata: " + "unknown blockette length for type %d", + blkt_type); + break; + } + + /* Make sure blockette is contained within the msrecord buffer */ + if ( (int)(blkt_offset - 4 + blkt_length) > size ) { + LOG_SID(CAPS_DEBUG, "read metadata: blockette " + "%d extends beyond record size, truncated?", + blkt_type); + break; + } + + if ( blkt_type == 1000 ) { + switch ( (int)bhead[4] ) { + case DE_ASCII: + header.dataType = DT_INT8; + break; + case DE_INT16: + header.dataType = DT_INT16; + break; + case DE_INT32: + case DE_STEIM1: + case DE_STEIM2: + case DE_CDSN: + case DE_DWWSSN: + case DE_SRO: + header.dataType = DT_INT32; + break; + case DE_FLOAT32: + header.dataType = DT_FLOAT; + break; + case DE_FLOAT64: + header.dataType = DT_DOUBLE; + break; + case DE_GEOSCOPE24: + case DE_GEOSCOPE163: + case DE_GEOSCOPE164: + header.dataType = DT_FLOAT; + break; + default: + break; + } + } + else if ( blkt_type == 1001 ) { + // Add usec correction + startTime += TimeSpan(0, *reinterpret_cast(bhead + 5)); + } + + /* Check that the next blockette offset is beyond the current blockette */ + if ( next_blkt && next_blkt < (blkt_offset + blkt_length - 4) ) { + LOG_SID(CAPS_DEBUG, "read metadata: offset to " + "next blockette (%d) is within current blockette " + "ending at byte %d", + blkt_type, (blkt_offset + blkt_length - 4)); + break; + } + /* Check that the offset is within record length */ + else if ( next_blkt && next_blkt > size ) { + LOG_SID(CAPS_DEBUG, "read metadata: offset to " + "next blockette (%d) from type %d is beyond record " + "length", next_blkt, blkt_type); + break; + } + else + blkt_offset = next_blkt; + } + + endTime = startTime; + + if ( *pMS2FSDH_SAMPLERATEFACT(head) > 0 ) { + header.samplingFrequencyNumerator = *pMS2FSDH_SAMPLERATEFACT(head); + header.samplingFrequencyDenominator = 1; + } + else { + header.samplingFrequencyNumerator = 1; + header.samplingFrequencyDenominator = -*pMS2FSDH_SAMPLERATEFACT(head); + } + + if ( *pMS2FSDH_SAMPLERATEMULT(head) > 0 ) { + header.samplingFrequencyNumerator *= *pMS2FSDH_SAMPLERATEMULT(head); + } + else { + header.samplingFrequencyDenominator *= -*pMS2FSDH_SAMPLERATEMULT(head); + } + + if ( header.samplingFrequencyNumerator > 0.0 && *pMS2FSDH_NUMSAMPLES(head) > 0 ) { + int64_t dt = static_cast(*pMS2FSDH_NUMSAMPLES(head)) * 1000000 * header.samplingFrequencyDenominator / header.samplingFrequencyNumerator; + endTime += TimeSpan(0, dt); + } + + timeToTimestamp(header.samplingTime, startTime); + } + else if ( MS3_ISVALIDHEADER(((char*)&head)) ) { + startTime = Time::FromYearDay( + Endianess::Converter::ToLittleEndian(*pMS3FSDH_YEAR(head)), + Endianess::Converter::ToLittleEndian(*pMS3FSDH_DAY(head)) + ); + + startTime += TimeSpan( + Endianess::Converter::ToLittleEndian(*pMS3FSDH_HOUR(head)) * 3600 + + Endianess::Converter::ToLittleEndian(*pMS3FSDH_MIN(head)) * 60 + + Endianess::Converter::ToLittleEndian(*pMS3FSDH_SEC(head)), + Endianess::Converter::ToLittleEndian(*pMS3FSDH_NSEC(head)) / 1000 + ); + + auto nsamp = Endianess::Converter::ToLittleEndian(*pMS3FSDH_NUMSAMPLES(head)); + auto fsamp = Endianess::Converter::ToLittleEndian(*pMS3FSDH_SAMPLERATE(head)); + if ( fsamp < 0 ) { + fsamp = -1.0 / fsamp; + } + auto encoding = Endianess::Converter::ToLittleEndian(*pMS3FSDH_ENCODING(head)); + + header.dataType = DT_Unknown; + switch ( encoding ) { + case DE_ASCII: + header.dataType = DT_INT8; + break; + case DE_INT16: + header.dataType = DT_INT16; + break; + case DE_INT32: + case DE_STEIM1: + case DE_STEIM2: + case DE_CDSN: + case DE_DWWSSN: + case DE_SRO: + header.dataType = DT_INT32; + break; + case DE_FLOAT32: + header.dataType = DT_FLOAT; + break; + case DE_FLOAT64: + header.dataType = DT_DOUBLE; + break; + case DE_GEOSCOPE24: + case DE_GEOSCOPE163: + case DE_GEOSCOPE164: + header.dataType = DT_FLOAT; + break; + default: + break; + } + + header.quality.ID = 0; + + if ( !double2frac(header.samplingFrequencyNumerator, header.samplingFrequencyDenominator, fsamp) ) { + CAPS_WARNING("read metadata: invalid sampling rate: %f", fsamp); + return false; + } + + endTime = startTime; + if ( header.samplingFrequencyNumerator > 0.0 && *pMS3FSDH_NUMSAMPLES(head) > 0 ) { + int64_t dt = static_cast(nsamp) * 1000000 * header.samplingFrequencyDenominator / header.samplingFrequencyNumerator; + endTime += TimeSpan(0, dt); + } + + timeToTimestamp(header.samplingTime, startTime); + return false; + } + else { CAPS_WARNING("read metadata: invalid MSEED header"); return false; } - bool headerswapflag = false; - if ( !MS_ISVALIDYEARDAY(head.start_time.year, head.start_time.day) ) - headerswapflag = true; - - /* Swap byte order? */ - if ( headerswapflag ) { - MS_SWAPBTIME(&head.start_time); - ms_gswap2a(&head.numsamples); - ms_gswap2a(&head.samprate_fact); - ms_gswap2a(&head.samprate_mult); - ms_gswap4a(&head.time_correct); - ms_gswap2a(&head.data_offset); - ms_gswap2a(&head.blockette_offset); - } - - header.dataType = DT_Unknown; - - hptime_t hptime = ms_btime2hptime(&head.start_time); - if ( hptime == HPTERROR ) { - LOG_SID(CAPS_DEBUG, "read metadata: invalid start time"); - return false; - } - - header.quality.ID = 0; - header.quality.str[0] = head.dataquality; - - if ( head.time_correct != 0 && !(head.act_flags & 0x02) ) - hptime += (hptime_t)head.time_correct * (HPTMODULUS / 10000); - - // Parse blockettes - uint32_t blkt_offset = head.blockette_offset; - uint32_t blkt_length; - uint16_t blkt_type; - uint16_t next_blkt; - - if ( blkt_offset < sizeof(head) ) { - LOG_SID(CAPS_DEBUG, "read metadata: blockette " - "offset points into header"); - return false; - } - - uint32_t coffs = 0; - - while ( (blkt_offset != 0) && ((int)blkt_offset < size) && - (blkt_offset < MAXRECLEN) ) { - char bhead[6]; - int seek_ofs = blkt_offset-sizeof(head)-coffs; - buf.pubseekoff(seek_ofs, std::ios_base::cur, std::ios_base::in); - coffs += seek_ofs; - if ( buf.sgetn(bhead, 6) != 6 ) { - LOG_SID(CAPS_DEBUG, "read metadata: " - "failed to read blockette header"); - break; - } - - coffs += 6; - - memcpy(&blkt_type, bhead, 2); - memcpy(&next_blkt, bhead+2, 2); - - if ( headerswapflag ) { - ms_gswap2(&blkt_type); - ms_gswap2(&next_blkt); - } - - blkt_length = ms_blktlen(blkt_type, bhead, headerswapflag); - - if ( blkt_length == 0 ) { - LOG_SID(CAPS_DEBUG, "read metadata: " - "unknown blockette length for type %d", - blkt_type); - break; - } - - /* Make sure blockette is contained within the msrecord buffer */ - if ( (int)(blkt_offset - 4 + blkt_length) > size ) { - LOG_SID(CAPS_DEBUG, "read metadata: blockette " - "%d extends beyond record size, truncated?", - blkt_type); - break; - } - - if ( blkt_type == 1000 ) { - switch ( (int)bhead[4] ) { - case DE_ASCII: - header.dataType = DT_INT8; - break; - case DE_INT16: - header.dataType = DT_INT16; - break; - case DE_INT32: - case DE_STEIM1: - case DE_STEIM2: - case DE_CDSN: - case DE_DWWSSN: - case DE_SRO: - header.dataType = DT_INT32; - break; - case DE_FLOAT32: - header.dataType = DT_FLOAT; - break; - case DE_FLOAT64: - header.dataType = DT_DOUBLE; - break; - case DE_GEOSCOPE24: - case DE_GEOSCOPE163: - case DE_GEOSCOPE164: - header.dataType = DT_FLOAT; - break; - default: - break; - } - } - else if ( blkt_type == 1001 ) { - // Add usec correction - hptime += ((hptime_t)bhead[5]) * (HPTMODULUS / 1000000); - } - - /* Check that the next blockette offset is beyond the current blockette */ - if ( next_blkt && next_blkt < (blkt_offset + blkt_length - 4) ) { - LOG_SID(CAPS_DEBUG, "read metadata: offset to " - "next blockette (%d) is within current blockette " - "ending at byte %d", - blkt_type, (blkt_offset + blkt_length - 4)); - break; - } - /* Check that the offset is within record length */ - else if ( next_blkt && next_blkt > size ) { - LOG_SID(CAPS_DEBUG, "read metadata: offset to " - "next blockette (%d) from type %d is beyond record " - "length", next_blkt, blkt_type); - break; - } - else - blkt_offset = next_blkt; - } - - startTime = Time((hptime_t)hptime/HPTMODULUS,(hptime_t)hptime%HPTMODULUS); - endTime = startTime; - - if ( head.samprate_fact > 0 ) { - header.samplingFrequencyNumerator = head.samprate_fact; - header.samplingFrequencyDenominator = 1; - } - else { - header.samplingFrequencyNumerator = 1; - header.samplingFrequencyDenominator = -head.samprate_fact; - } - - if ( head.samprate_mult > 0 ) - header.samplingFrequencyNumerator *= head.samprate_mult; - else - header.samplingFrequencyDenominator *= -head.samprate_mult; - - if ( header.samplingFrequencyNumerator > 0.0 && head.numsamples > 0 ) { - hptime = (hptime_t)head.numsamples * HPTMODULUS * header.samplingFrequencyDenominator / header.samplingFrequencyNumerator; - endTime += TimeSpan((hptime_t)hptime/HPTMODULUS,(hptime_t)hptime%HPTMODULUS); - } - - timeToTimestamp(header.samplingTime, startTime); - return true; } @@ -319,68 +512,19 @@ bool MSEEDDataRecord::put(std::streambuf &buf, bool /*withHeader*/) const { return buf.sputn(&_data[0], _data.size()) == (int)_data.size(); } + void MSEEDDataRecord::setData(const void *data, size_t size) { _data.resize(size); memcpy(_data.data(), data, size); unpackHeader(); } -void MSEEDDataRecord::unpackHeader(char *data, size_t size) { - // Only unpack the header structure - MSRecord *ms_rec = NULL; - int state = msr_unpack(data, size, &ms_rec, 0, 0); - if ( state != MS_NOERROR ) { - CAPS_WARNING("read metadata: read error: %d", state); - if ( ms_rec != NULL ) - msr_free(&ms_rec); - return; + +void MSEEDDataRecord::unpackHeader() { + arraybuf tmp(_data.data(), _data.size()); + if ( !readMetaData(tmp, _data.size(), _header, _startTime, _endTime) ) { + CAPS_WARNING("invalid MSEED header"); } - - hptime_t hptime = msr_starttime(ms_rec); - _startTime = Time((hptime_t)hptime/HPTMODULUS,(hptime_t)hptime%HPTMODULUS); - _endTime = _startTime; - - if ( ms_rec->samprate > 0.0 && ms_rec->samplecnt > 0 ) { - hptime = (hptime_t)(((double)(ms_rec->samplecnt) / ms_rec->samprate * HPTMODULUS) + 0.5); - _endTime += TimeSpan((hptime_t)hptime/HPTMODULUS,(hptime_t)hptime%HPTMODULUS); - } - - _header.dataType = DT_Unknown; - timeToTimestamp(_header.samplingTime, _startTime); - - if ( ms_rec->fsdh->samprate_fact > 0 ) { - _header.samplingFrequencyNumerator = ms_rec->fsdh->samprate_fact; - _header.samplingFrequencyDenominator = 1; - } - else { - _header.samplingFrequencyNumerator = 1; - _header.samplingFrequencyDenominator = -ms_rec->fsdh->samprate_fact; - } - - if ( ms_rec->fsdh->samprate_mult > 0 ) - _header.samplingFrequencyNumerator *= ms_rec->fsdh->samprate_mult; - else - _header.samplingFrequencyDenominator *= -ms_rec->fsdh->samprate_mult; - - switch ( ms_rec->sampletype ) { - case 'a': - _header.dataType = DT_INT8; - break; - case 'i': - _header.dataType = DT_INT32; - break; - case 'f': - _header.dataType = DT_FLOAT; - break; - case 'd': - _header.dataType = DT_DOUBLE; - break; - default: - _header.dataType = DT_Unknown; - break; - } - - msr_free(&ms_rec); } diff --git a/libs/gempa/caps/mseedpacket.h b/libs/gempa/caps/mseedpacket.h index 37518f9..3d1efc9 100644 --- a/libs/gempa/caps/mseedpacket.h +++ b/libs/gempa/caps/mseedpacket.h @@ -18,7 +18,6 @@ #include -#include namespace Gempa { @@ -29,46 +28,47 @@ class MSEEDDataRecord : public DataRecord { public: MSEEDDataRecord(); - virtual const char *formatName() const; + DataRecord *clone() const override; + const char *formatName() const override; - virtual bool readMetaData(std::streambuf &buf, int size, - Header &header, - Time &startTime, - Time &endTime); + bool readMetaData(std::streambuf &buf, int size, + Header &header, + Time &startTime, + Time &endTime) override; - virtual const Header *header() const; - virtual Time startTime() const; - virtual Time endTime() const; + const Header *header() const override; + Time startTime() const override; + Time endTime() const override; - virtual bool canTrim() const; - virtual bool canMerge() const; + bool canTrim() const override; + bool canMerge() const override; - virtual bool trim(const Time &start, - const Time &end) const; + bool trim(const Time &start, + const Time &end) const override; - virtual size_t dataSize(bool withHeader) const; + size_t dataSize(bool withHeader) const override; - virtual ReadStatus get(std::streambuf &buf, int size, - const Time &start = Time(), - const Time &end = Time(), - int maxSize = -1); + ReadStatus get(std::streambuf &buf, int size, + const Time &start = Time(), + const Time &end = Time(), + int maxSize = -1) override; - virtual bool put(std::streambuf &buf, bool withHeader) const; + bool put(std::streambuf &buf, bool withHeader) const override; /** * @brief Returns the packet type * @return The packet type */ - PacketType packetType() const { return MSEEDPacket; } + PacketType packetType() const override { return MSEEDPacket; } /** * @brief Initializes the internal data vector from the given buffer * @param The buffer to read the data from * @param The buffer size */ - virtual void setData(const void *data, size_t size); + void setData(const void *data, size_t size); - void unpackHeader() { unpackHeader(_data.data(), _data.size()); } + void unpackHeader(); protected: @@ -78,10 +78,6 @@ class MSEEDDataRecord : public DataRecord { Time _endTime; int _dataType; - - - private: - void unpackHeader(char *data, size_t size); }; diff --git a/libs/gempa/caps/packet.h b/libs/gempa/caps/packet.h index 7a20864..23161e1 100644 --- a/libs/gempa/caps/packet.h +++ b/libs/gempa/caps/packet.h @@ -16,6 +16,7 @@ #ifndef GEMPA_CAPS_PACKET_H #define GEMPA_CAPS_PACKET_H + #include #include @@ -24,12 +25,11 @@ #include -#include +#include #include #include - namespace Gempa { namespace CAPS { @@ -320,6 +320,7 @@ class SC_GEMPA_CAPS_API DataRecord { public: virtual ~DataRecord(); + virtual DataRecord *clone() const = 0; virtual const char *formatName() const = 0; virtual bool readMetaData(std::streambuf &buf, int size, @@ -393,7 +394,8 @@ class SC_GEMPA_CAPS_API DataRecord { Buffer _data; }; -typedef boost::shared_ptr DataRecordPtr; + +using DataRecordPtr = boost::shared_ptr; struct RawPacket { @@ -403,6 +405,7 @@ struct RawPacket { DataRecord *record; }; + struct MetaPacket { std::string SID[4]; PacketDataHeader packetDataHeader; @@ -413,6 +416,7 @@ struct MetaPacket { Time timestamp; }; + } } diff --git a/libs/gempa/caps/plugin.cpp b/libs/gempa/caps/plugin.cpp index 518e7b1..6a4c2a1 100644 --- a/libs/gempa/caps/plugin.cpp +++ b/libs/gempa/caps/plugin.cpp @@ -38,17 +38,17 @@ #include #endif -#include - -#if !defined(CAPS_FEATURES_JOURNAL) || CAPS_FEATURES_JOURNAL -#include +#if !defined(CAPS_SC_LOGGING) || CAPS_SC_LOGGING +#include #endif +#include + #include #include +#include #include -#include #include #include @@ -56,39 +56,46 @@ using namespace std; #if !defined(CAPS_FEATURES_JOURNAL) || CAPS_FEATURES_JOURNAL -namespace fs = boost::filesystem; -#endif - -//see, http://www.boost.org/doc/libs/1_51_0/libs/filesystem/doc/deprecated.html -#if BOOST_FILESYSTEM_VERSION >= 3 - #define BOOST_FILESYSTEM_NO_DEPRECATED - - // path - #define FS_PATH(PATH_STR) boost::filesystem::path(PATH_STR) - #define FS_DECLARE_PATH(NAME, PATH_STR) \ - boost::filesystem::path NAME(PATH_STR); - - #define FS_HAS_PARENT_PATH(PATH) PATH.has_parent_path() - #define FS_PARENT_PATH(PATH) PATH.parent_path() - -#else - // path - #define FS_PATH(PATH_STR) boost::filesystem::path(PATH_STR,\ - boost::filesystem::native) - #define FS_DECLARE_PATH(NAME, PATH_STR) \ - boost::filesystem::path NAME(PATH_STR, boost::filesystem::native); - - #if BOOST_VERSION < 103600 - #define FS_HAS_PARENT_PATH(PATH) PATH.has_branch_path() - #define FS_PARENT_PATH(PATH) PATH.branch_path() - #else - #define FS_HAS_PARENT_PATH(PATH) PATH.has_parent_path() - #define FS_PARENT_PATH(PATH) PATH.parent_path() - #endif +namespace fs = std::filesystem; #endif namespace { +#if !defined(CAPS_SC_LOGGING) || CAPS_SC_LOGGING +#define LOG_CHANNEL(out, fmt) \ + va_list ap;\ + va_start(ap, fmt);\ + out(fmt, ap);\ + va_end(ap) +#else +#define LOG_CHANNEL(out, fmt) \ + va_list ap;\ + va_start(ap, fmt);\ + fprintf(stderr, #out" "); vfprintf(stderr, fmt, ap); fprintf(stderr, "\n");\ + va_end(ap) +#endif + +#if !defined(CAPS_SC_LOGGING) || CAPS_SC_LOGGING +void LogError(const char *fmt, ...) { + LOG_CHANNEL(SEISCOMP_VERROR, fmt); +} + +void LogWarning(const char *fmt, ...) { + LOG_CHANNEL(SEISCOMP_VWARNING, fmt); +} + +void LogNotice(const char *fmt, ...) { + LOG_CHANNEL(SEISCOMP_VNOTICE, fmt); +} + +void LogInfo(const char *fmt, ...) { + LOG_CHANNEL(SEISCOMP_VINFO, fmt); +} + +void LogDebug(const char *fmt, ...) { + LOG_CHANNEL(SEISCOMP_VDEBUG, fmt); +} +#else #define LOG_CHANNEL(out, fmt) \ va_list ap;\ va_start(ap, fmt);\ @@ -114,6 +121,45 @@ void LogInfo(const char *fmt, ...) { void LogDebug(const char *fmt, ...) { LOG_CHANNEL(DEBUG, fmt); } +#endif + +/** + * @brief Validate stream ID components regarding the following + * pattern [a-z][A-Z][0-9][-,_] + * @param net The network code + * @param sta The station code + * @param loc The location code + * @param cha The channel code + * @return True on success + */ +bool validateStreamID(const std::string &net, const std::string &sta, + const std::string &loc, const std::string &cha) { + std::vector SID = {&net, &sta, &loc, &cha}; + for ( int i = 0; i < 4; ++i ) { + for ( const char *ch = SID[i]->c_str(); *ch != 0; ++ch ) { + if ( (*ch < 65 || *ch > 90) && // A-Z + (*ch < 48 || *ch > 57) && // 0-9 + (*ch < 97 || *ch > 122) && // a-z + *ch != 45 && *ch != 95 ) { // -, _ + CAPS_ERROR("%s.%s.%s.%s: Invalid character in token #%i (%s) of " + "stream id", net.c_str(), sta.c_str(), loc.c_str(), + cha.c_str(), i, SID[i]->c_str()); + return false; + } + } + } + + return true; +} + +template +inline std::string toString(const T &v) { + std::ostringstream os; + os << v; + return os.str(); +} + + } @@ -127,10 +173,12 @@ namespace { #if !defined(CAPS_FEATURES_JOURNAL) || CAPS_FEATURES_JOURNAL bool createPath(const string &dir) { try { - FS_DECLARE_PATH(path, dir) - fs::is_directory(path); - fs::create_directories(path); - return true; + fs::path path(dir); + if ( fs::is_directory(path) ) { + return true; + } + + return fs::create_directories(path); } catch ( ... ) { return false; } @@ -280,9 +328,8 @@ Plugin::Plugin(const string &name, const string &options, : _name(name) , _options(options) , _description(description) -, _bufferSize(1 << 14) +, _bufferSize(1 << 17) , _bytesBuffered(0) -, _sendTimeout(60) , _isExitRequested(false) #if !defined(CAPS_FEATURES_BACKFILLING) || CAPS_FEATURES_BACKFILLING , _backfillingBufferSize(0) @@ -299,6 +346,9 @@ Plugin::Plugin(const string &name, const string &options, _wasConnected = false; _ackTimeout = 5; _lastAckTimeout = 5; + _sendTimeout = 60; + _readTimeout = _sendTimeout; + _connectionTimeout = 30; _encoderFactory = NULL; _maxFutureEndTime = TimeSpan(120, 0); #if !defined(CAPS_FEATURES_SSL) || CAPS_FEATURES_SSL @@ -310,7 +360,6 @@ Plugin::Plugin(const string &name, const string &options, Plugin::~Plugin() { if ( !_closed ) close(); - setEncoderFactory(NULL); } @@ -321,6 +370,10 @@ void Plugin::close() { #endif _closed = true; + CAPS_INFO("[%p] Stop sending data requested", static_cast(this)); + + CAPS_INFO("[%p] Flushing backfilling buffer", static_cast(this)); + for ( StreamStates::iterator it = _states.begin(); it != _states.end(); ++it ) { StreamState &state = it->second; @@ -332,10 +385,13 @@ void Plugin::close() { } } + CAPS_INFO("[%p] Flushing %zu encoders", + static_cast(this), _encoderItems.size()); flushEncoders(); sendBye(); - CAPS_INFO("Closing connection to CAPS at %s:%d", _url.host.c_str(), _url.port); + CAPS_INFO("[%p] Closing connection to CAPS at %s:%d", + static_cast(this), _url.host.c_str(), _url.port); while ( !_packetBuffer.empty() && readResponse(_lastAckTimeout) ); @@ -344,13 +400,15 @@ void Plugin::close() { #endif disconnect(); - CAPS_INFO("Closed connection to CAPS at %s:%d", _url.host.c_str(), _url.port); + CAPS_INFO("[%p] Closed connection to CAPS at %s:%d", + static_cast(this), _url.host.c_str(), _url.port); _packetBuffer.clear(); _wasConnected = false; } void Plugin::quit() { + CAPS_INFO("[%p] Shutdown was requested from outside", static_cast(this)); _isExitRequested = true; } @@ -361,26 +419,59 @@ bool Plugin::connect() { _socket = SocketPtr(new CAPS::Socket()); #endif - CAPS_INFO("Attempting to connect to CAPS at %s:%d", - _url.host.c_str(), _url.port); - if ( _socket->connect(_url.host, _url.port) != Socket::Success ) { - CAPS_ERROR("Connection failed to CAPS at %s:%d", _url.host.c_str(), _url.port); + CAPS_INFO("[%p] Attempting to connect to CAPS at %s:%d", + static_cast(this), _url.host.c_str(), _url.port); + auto status = _socket->connect(_url.host, _url.port, _connectionTimeout); + if ( status != Socket::Success ) { + CAPS_ERROR("[%p] Connection failed to CAPS at %s:%d", + static_cast(this), _url.host.c_str(), _url.port); return false; } // Do handshake - int apiVersion = 0; if ( !getAPIVersion(apiVersion) ) { return false; } - CAPS_INFO("Found CAPS API version %d", apiVersion); + CAPS_INFO("[%p] Found CAPS API version %d", static_cast(this), apiVersion); if ( apiVersion >= 5 ) { - if ( !_agent.empty() ) { - _socket->write("AGENT ", 6); - _socket->write(_agent.data(), _agent.size()); - _socket->write("\n", 1); + if ( !_hostInfo.agent.empty() ) { + _socket->send("AGENT "); + _socket->send(_hostInfo.agent.data()); + _socket->send("\n"); + } + } + + if ( apiVersion >= 7 ) { + if ( !_hostInfo.agentVersion.empty() ) { + _socket->send("VERSION "); + _socket->send(_hostInfo.agentVersion.data()); + _socket->send("\n"); + } + + if ( !_hostInfo.os.empty() ) { + _socket->send("OS "); + _socket->send(_hostInfo.os.data()); + _socket->send("\n"); + } + + if ( _hostInfo.totalDisc != -1 ) { + _socket->send("TOTALDISC "); + _socket->send(toString(_hostInfo.totalDisc).c_str()); + _socket->send("\n"); + } + + if ( _hostInfo.totalMem != -1 ) { + _socket->send("TOTALMEM "); + _socket->send(toString(_hostInfo.totalMem).c_str()); + _socket->send("\n"); + } + + sendRuntimeInfo(); + + if ( !getConnectionID() ) { + return false; } } @@ -400,7 +491,8 @@ bool Plugin::connect() { FD_SET(_socket->fd(), &_readFDs); FD_SET(_socket->fd(), &_writeFDs); - CAPS_INFO("Connected to CAPS at %s:%d", _url.host.c_str(), _url.port); + CAPS_INFO("[%p] Connected to CAPS at %s:%d", + static_cast(this), _url.host.c_str(), _url.port); _responseBuf[0] = '\0'; _responseBufIdx = 0; @@ -416,17 +508,26 @@ bool Plugin::connect() { _wasConnected = true; + if ( result && _connectedCallback ) { + _connectedCallback(); + } + return result; } void Plugin::disconnect() { if ( _socket && _socket->isValid() ) { - CAPS_INFO("Disconnect from %s:%d", _url.host.c_str(), _url.port); + CAPS_INFO("[%p] Disconnect from %s:%d", + static_cast(this), _url.host.c_str(), _url.port); _socket->shutdown(); _socket->close(); _responseBuf[0] = '\0'; _responseBufIdx = 0; + + if ( _disconnectedCallback ) { + _disconnectedCallback(); + } } } @@ -463,13 +564,14 @@ bool Plugin::readResponse(unsigned int timeout) { res = _socket->read(buf, bufN); if ( res < 0 ) { if ( errno != EAGAIN && errno != EWOULDBLOCK ) { - CAPS_ERROR("Reading failed: %s: disconnect", strerror(errno)); + CAPS_ERROR("[%p] Reading failed: %s: disconnect", + static_cast(this), strerror(errno)); disconnect(); } break; } else if ( res == 0 ) { - CAPS_INFO("Peer closed connection"); + CAPS_INFO("[%p] Peer closed connection", static_cast(this)); disconnect(); break; } @@ -486,16 +588,20 @@ bool Plugin::readResponse(unsigned int timeout) { // Read confirmed packets from response int count = atoi(_responseBuf+3); - CAPS_DEBUG("Acknowledged %d packets, %zu in queue", count, _packetBuffer.size()); + CAPS_DEBUG("[%p] Acknowledged %d packets, %zu in queue", + static_cast(this), count, _packetBuffer.size()); // Update packet buffer for ( int i = 0; i < count; ++i ) { if ( _packetBuffer.empty() ) { - CAPS_ERROR("Synchronization error: more packages acknowledged than in queue"); + CAPS_ERROR("[%p] Synchronization error: more packages acknowledged than in queue", + static_cast(this)); break; } PacketPtr packet = _packetBuffer.front(); - _states[packet->streamID].lastEndTime = packet->record->endTime(); + if ( packet->record->endTime() > _states[packet->streamID].lastEndTime ) { + _states[packet->streamID].lastEndTime = packet->record->endTime(); + } _packetBuffer.pop_front(); _bytesBuffered -= packet->record->dataSize(); _packetBufferDirty = true; @@ -504,20 +610,22 @@ bool Plugin::readResponse(unsigned int timeout) { #endif gotResponse = true; - CAPS_DEBUG("Packet acknowledged by CAPS, stream: %s' time window: %s~%s", + CAPS_DEBUG("[%p] Packet acknowledged by CAPS, stream: %s' time window: %s~%s", + static_cast(this), packet->streamID.c_str(), packet->record->startTime().iso().c_str(), packet->record->endTime().iso().c_str()); if ( _packetAckFunc ) { - _packetAckFunc(packet->streamID, packet->record->startTime(), packet->record->endTime()); + _packetAckFunc(packet->streamID, packet->record->startTime(), packet->record->endTime(), packet->context); } } - CAPS_DEBUG("Packet buffer state: %zu packets, %zu bytes", + CAPS_DEBUG("[%p] Packet buffer state: %zu packets, %zu bytes", + static_cast(this), _packetBuffer.size(), _bytesBuffered); } else if ( (strncasecmp(_responseBuf, "ERROR ", 6) == 0) && (_responseBufIdx > 6) ) { - CAPS_ERROR("%s", _responseBuf + 6); + CAPS_ERROR("[%p] %s", static_cast(this), _responseBuf + 6); return false; } @@ -559,7 +667,6 @@ bool Plugin::readJournal() { bool Plugin::readJournal(istream &is) { _states.clear(); - string streamID, strTime; string line; Time time; int lineNumber = 0; @@ -569,26 +676,44 @@ bool Plugin::readJournal(istream &is) { while ( getline(is, line) ) { ++lineNumber; - size_t p = line.find(' '); - if ( p == string::npos ) - streamID = trim(line); - else { - streamID = trim(line.substr(0, p)); - strTime = trim(line.substr(p+1)); + trim(line); + if ( line.empty() ) { + continue; + } + + if ( line[0] == '#' ) { + continue; + } + + std::vector toks; + boost::split(toks, line, boost::is_any_of(" ")); + if ( toks.empty() ) { + CAPS_ERROR("[%p] journal:%d: Invalid line: %s", + static_cast(this), lineNumber, line.c_str()); + return false; + } + + std::string streamID = toks[0]; + string strTime; + if ( toks.size() > 1 ) { + strTime = toks[1]; } if ( !strTime.empty() ) { if ( !time.fromString(strTime.c_str(), "%FT%T.%fZ") ) { - CAPS_ERROR("journal:%d: Invalid time: %s", lineNumber, strTime.c_str()); + CAPS_ERROR("[%p] journal:%d: Invalid time: %s", + static_cast(this), lineNumber, strTime.c_str()); return false; } } - else + else { time = Time(); + } if ( time > maxAllowedEndTime ) { - CAPS_WARNING("journal:%d:%s: Timestamp %s is more than one day " + CAPS_WARNING("[%p] journal:%d:%s: Timestamp %s is more than one day " "ahead of current time, respecting it nevertheless.", + static_cast(this), lineNumber, streamID.c_str(), time.iso().c_str()); } @@ -597,10 +722,21 @@ bool Plugin::readJournal(istream &is) { return true; } + +Gempa::CAPS::Time Plugin::lastEndTime(const std::string &id) { + Gempa::CAPS::Plugin::StreamStates::const_iterator it = _states.find(id); + if ( it == _states.end() ) { + return Gempa::CAPS::Time(); + } + + return it->second.lastEndTime; +} + #endif bool Plugin::flush() { - CAPS_INFO("Flushing %zu queued packets", _packetBuffer.size()); + CAPS_INFO("[%p] Flushing %zu queued packets", static_cast(this), + _packetBuffer.size()); PacketBuffer::iterator it = _packetBuffer.begin(); while ( it != _packetBuffer.end() && !_isExitRequested ) { @@ -609,7 +745,8 @@ bool Plugin::flush() { if ( !sendPacket(packet.get() ) && !_isExitRequested ) { if ( _packetBufferDirty ) { - CAPS_ERROR("Uh oh, buffer dirty but sending failed!"); + CAPS_ERROR("[%p] Uh oh, buffer dirty but sending failed!", + static_cast(this)); } return false; @@ -626,7 +763,8 @@ bool Plugin::flush() { if ( it != _packetBuffer.end() ) ++it; else { - CAPS_DEBUG("Last packet removed, reset flush queue iterator"); + CAPS_DEBUG("[%p] Last packet removed, reset flush queue iterator", + static_cast(this)); it = _packetBuffer.begin(); } } @@ -638,7 +776,6 @@ bool Plugin::flush() { } void Plugin::flushEncoders() { - CAPS_INFO("Flushing %zu encoders", _encoderItems.size()); EncoderItems::iterator it = _encoderItems.begin(); while ( it != _encoderItems.end() ) { Encoder *encoder = it->second.encoder.get(); @@ -661,9 +798,23 @@ Plugin::Status Plugin::push(const string &net, const string &sta, uint16_t numerator, uint16_t denominator, const string &uom, void *data, size_t count, DataType dt, - int timingQuality) { + int timingQuality, void *context) { + return pushRaw(net, sta, loc, cha, stime, numerator, denominator, uom, + data, count, dt, timingQuality, context); +} + + +Plugin::Status Plugin::pushRaw(const string &net, const string &sta, + const string &loc, const string &cha, + const Time &stime, + uint16_t numerator, uint16_t denominator, + const string &uom, + void *data, size_t count, DataType dt, + int timingQuality, void *context) { uint8_t dtSize = dataTypeSize(dt); - if ( dtSize == 0 ) return Plugin::PacketLoss; + if ( !dtSize ) { + return Plugin::PacketLoss; + } RawDataRecord *rec = new RawDataRecord(); rec->setStartTime(stime); @@ -671,7 +822,7 @@ Plugin::Status Plugin::push(const string &net, const string &sta, rec->setDataType(dt); rec->setBuffer(static_cast(data), dtSize * count); - return push(net, sta, loc, cha, DataRecordPtr(rec), uom, timingQuality); + return push(net, sta, loc, cha, DataRecordPtr(rec), uom, timingQuality, context); } #if !defined(CAPS_FEATURES_ANY) || CAPS_FEATURES_ANY @@ -679,7 +830,7 @@ Plugin::Status Plugin::push(const string &net, const std::string &sta, const std::string &loc, const std::string &cha, const Time &stime, uint16_t numerator, uint16_t denominator, const std::string &format, - char *data, size_t count) { + char *data, size_t count, void *context) { AnyDataRecord *rec = new AnyDataRecord; rec->setStartTime(stime); rec->setEndTime(stime); @@ -687,16 +838,43 @@ Plugin::Status Plugin::push(const string &net, const std::string &sta, rec->setType(format.c_str()); rec->setData(data, count); - return push(net, sta, loc, cha, DataRecordPtr(rec), "px"); + return push(net, sta, loc, cha, DataRecordPtr(rec), "px", -1, context); +} + +Plugin::Status Plugin::pushAny(const string &net, const std::string &sta, + const std::string &loc, const std::string &cha, + const Time &stime, const Time &etime, + uint16_t numerator, uint16_t denominator, + const std::string &format, const std::string &uom, + char *data, size_t count, void *context) { + AnyDataRecord *rec = new AnyDataRecord; + rec->setStartTime(stime); + rec->setEndTime(etime); + rec->setSamplingFrequency(numerator, denominator); + rec->setType(format.c_str()); + rec->setData(data, count); + + return push(net, sta, loc, cha, DataRecordPtr(rec), uom, -1, context); } Plugin::Status Plugin::push(const string &net, const std::string &sta, const std::string &loc, const std::string &cha, const Time &stime, uint16_t numerator, uint16_t denominator, const std::string &format, - const std::string &str) { + const std::string &str, void *context) { return push(net, sta, loc, cha, stime, numerator, denominator, format, - const_cast(&str[0]), str.size()); + const_cast(str.data()), str.size(), context); +} + +Plugin::Status Plugin::pushAny(const string &net, const std::string &sta, + const std::string &loc, const std::string &cha, + const Time &stime, const Time &etime, + uint16_t numerator, uint16_t denominator, + const std::string &format, const std::string &uom, + const std::string &str, void *context) { + return pushAny(net, sta, loc, cha, stime, etime, + numerator, denominator, format, uom, + const_cast(str.data()), str.size(), context); } #endif @@ -723,13 +901,15 @@ void Plugin::tryFlushBackfillingBuffer(StreamState &state) { if ( flushed > 0 ) { if ( state.backfillingBuffer.size() > 0 ) { - CAPS_DEBUG("backfilling buffer: %zu pakets flushed, new size: %zu, time window: %s~%s", + CAPS_DEBUG("[%p] backfilling buffer: %zu pakets flushed, new size: %zu, time window: %s~%s", + static_cast(this), flushed, state.backfillingBuffer.size(), state.backfillingBuffer.front()->record->startTime().iso().c_str(), state.backfillingBuffer.back()->record->endTime().iso().c_str()); } else { - CAPS_DEBUG("backfilling buffer: %zu pakets flushed, new size: 0", + CAPS_DEBUG("[%p] backfilling buffer: %zu pakets flushed, new size: 0", + static_cast(this), flushed); } } @@ -754,14 +934,15 @@ void Plugin::trimBackfillingBuffer(StreamState &state) { if ( trimmed > 0 ) { if ( state.backfillingBuffer.size() > 0 ) { - CAPS_DEBUG("backfilling buffer: %zu pakets trimmed, new size: %zu, time window: %s~%s", + CAPS_DEBUG("[%p] backfilling buffer: %zu pakets trimmed, new size: %zu, time window: %s~%s", + static_cast(this), trimmed, state.backfillingBuffer.size(), state.backfillingBuffer.front()->record->startTime().iso().c_str(), state.backfillingBuffer.back()->record->endTime().iso().c_str()); } else { - CAPS_DEBUG("backfilling buffer: %zu pakets trimmed, new size: 0", - trimmed); + CAPS_DEBUG("[%p] backfilling buffer: %zu pakets trimmed, new size: 0", + static_cast(this), trimmed); } } } @@ -769,37 +950,59 @@ void Plugin::trimBackfillingBuffer(StreamState &state) { Plugin::Status Plugin::push(const string &net, const string &sta, const string &loc, const string &cha, DataRecordPtr rec, const string &uom, - int timingQuality) { + int timingQuality, void *context) { + return pushRecord(net, sta, loc, cha, rec, uom, timingQuality, context); +} + +Plugin::Status Plugin::pushRecord(const string &net, const string &sta, + const string &loc, const string &cha, + DataRecordPtr rec, const string &uom, + int timingQuality, void *context) { static bool showVersion = true; if ( showVersion ) { - CAPS_NOTICE("LIB CAPS version %s", version()); + CAPS_NOTICE("[%p] LIB CAPS version %s", static_cast(this), version()); showVersion = false; } - if ( rec == NULL ) return PacketNotValid; - if ( rec->dataSize() > _bufferSize ) return PacketSize; + if ( !rec ) { + return PacketNotValid; + } - if ( !rec->endTime().valid() ) return PacketNotValid; + if ( rec->dataSize() > _bufferSize ) { + return PacketSize; + } - Time endTime = getLastSampleTime(rec.get()); - Time maxAllowedEndTime = Time::GMT() + TimeSpan(_maxFutureEndTime); - if ( endTime > maxAllowedEndTime ) { - CAPS_WARNING("%s.%s.%s.%s: Future time stamp detected: Packet end time %s exceeds " - "max allowed end time %s. Discard packet.", - net.c_str(), sta.c_str(), loc.c_str(), cha.c_str(), - rec->endTime().iso().c_str(), - maxAllowedEndTime.iso().c_str()); - return MaxFutureEndTimeExceeded; + if ( !rec->endTime().valid() ) { + return PacketNotValid; + } + + if ( !validateStreamID(net, sta, loc, cha) ) { + return PacketNotValid; + } + + if ( _maxFutureEndTime.seconds() >= 0 ) { + Time endTime = getLastSampleTime(rec.get()); + Time maxAllowedEndTime = Time::GMT() + TimeSpan(_maxFutureEndTime); + if ( endTime > maxAllowedEndTime ) { + CAPS_WARNING("[%p] %s.%s.%s.%s: Future time stamp detected: Packet end time %s exceeds " + "max allowed end time %s. Discard packet.", + static_cast(this), + net.c_str(), sta.c_str(), loc.c_str(), cha.c_str(), + rec->endTime().iso().c_str(), + maxAllowedEndTime.iso().c_str()); + return MaxFutureEndTimeExceeded; + } } const DataRecord::Header *header = rec->header(); - PacketPtr packet(new Packet(rec, net, sta, loc, cha)); + PacketPtr packet(new Packet(rec, net, sta, loc, cha, context)); packet->uom = uom; packet->dataType = header->dataType; packet->timingQuality = timingQuality; #if !defined(CAPS_FEATURES_BACKFILLING) || CAPS_FEATURES_BACKFILLING if ( header->samplingFrequencyNumerator == 0 ) { - CAPS_DEBUG("detected division by zero, invalid sampling frequency numerator"); + CAPS_DEBUG("[%p] detected division by zero, invalid sampling frequency numerator", + static_cast(this)); return PacketNotValid; } @@ -819,7 +1022,8 @@ Plugin::Status Plugin::push(const string &net, const string &sta, // A gap larger than one sample? if ( ((int64_t)gap.seconds()*1000000+gap.microseconds()) >= dt_us ) { - CAPS_DEBUG("detected gap on stream: %s", packet->streamID.c_str()); + CAPS_DEBUG("[%p] detected gap on stream: %s", + static_cast(this), packet->streamID.c_str()); insertPacket(state.backfillingBuffer, packet); trimBackfillingBuffer(state); tryFlushBackfillingBuffer(state); @@ -838,8 +1042,9 @@ Plugin::Status Plugin::push(const string &net, const string &sta, } else { #endif - if ( !encodePacket(packet) ) - return PacketLoss; + if ( !encodePacket(packet) ) { + return PacketLoss; + } #if !defined(CAPS_FEATURES_BACKFILLING) || CAPS_FEATURES_BACKFILLING } #endif @@ -874,20 +1079,24 @@ bool Plugin::commitPacket(PacketPtr packet) { wait(); } - if ( !isConnected() ) return false; + if ( !isConnected() ) { + return false; + } } if ( _bytesBuffered >= _bufferSize ) { - CAPS_DEBUG("Packet buffer is full (%zu/%zu bytes), " + CAPS_DEBUG("[%p] Packet buffer is full (%zu/%zu bytes), " "waiting for server ack messages", + static_cast(this), _bytesBuffered, _bufferSize); while ( _bytesBuffered >= _bufferSize ) { if ( !readResponse(_ackTimeout) ) { - CAPS_WARNING("Packet buffer was full (%zu/%zu bytes), " - "did not receive ack within %d seconds", + CAPS_WARNING("[%p] Packet buffer was full (%zu/%zu bytes), " + "did not receive ack within %d seconds: %s", + static_cast(this), _bytesBuffered, _bufferSize, - _ackTimeout); + _ackTimeout, _isExitRequested ? "abort" : "reconnect"); disconnect(); while ( !_isExitRequested && !_socket->isValid() ) { wait(); @@ -900,25 +1109,28 @@ bool Plugin::commitPacket(PacketPtr packet) { } } - CAPS_DEBUG("%zu/%zu bytes buffered after force wait", + CAPS_DEBUG("[%p] %zu/%zu bytes buffered after force wait", + static_cast(this), _bytesBuffered, _bufferSize); - if ( _bytesBuffered >= _bufferSize ) + if ( _bytesBuffered >= _bufferSize ) { return false; + } #if !defined(CAPS_FEATURES_JOURNAL) || CAPS_FEATURES_JOURNAL - CAPS_DEBUG("Force journal update"); + CAPS_DEBUG("[%p] Force journal update", static_cast(this)); #endif } // Serialize data record serializePacket(packet.get()); - CAPS_DEBUG("+ buffer state: %zu packets, %zu bytes", + CAPS_DEBUG("[%p] + buffer state: %zu packets, %zu bytes", + static_cast(this), _packetBuffer.size(), _bytesBuffered); while ( !sendPacket(packet.get() ) ) { - CAPS_ERROR("Sending failed: %s", _isExitRequested ? "abort" : "reconnect"); + CAPS_ERROR("[%p] Sending failed: %s", static_cast(this), _isExitRequested ? "abort" : "reconnect"); readResponse(); disconnect(); @@ -929,8 +1141,9 @@ bool Plugin::commitPacket(PacketPtr packet) { connect(); } - if ( !isConnected() ) + if ( !isConnected() ) { return false; + } } _packetBuffer.push_back(packet); @@ -958,7 +1171,10 @@ Encoder* Plugin::getEncoder(PacketPtr packet) { const DataRecord::Header *header = record->header(); EncoderItem *item = &it->second; - if ( item->encoder != NULL ) { + if ( item->encoder ) { + // Copy context + item->encoder->setContext(packet->context); + /* Before we can feed data into the encoder we have to check if the sampling frequency or data type have been changed. Also gaps @@ -971,7 +1187,11 @@ Encoder* Plugin::getEncoder(PacketPtr packet) { needsFlush = true; } else if ( record->startTime() != clk.getTime(0) ) { - needsFlush = true; + auto gap = abs(static_cast(record->startTime() - clk.getTime(0))); + if ( (header->samplingFrequencyNumerator == 0) + || (gap > header->samplingFrequencyDenominator * 0.5 / header->samplingFrequencyNumerator) ) { + needsFlush = true; + } } else if ( item->dataType != header->dataType ) { needsFlush = true; @@ -993,14 +1213,15 @@ Encoder* Plugin::getEncoder(PacketPtr packet) { } } - if ( item->encoder == NULL ) { + if ( !item->encoder ) { Encoder *encoder = _encoderFactory->create(packet->networkCode, packet->stationCode, packet->locationCode, packet->channelCode, header->dataType, header->samplingFrequencyNumerator, header->samplingFrequencyDenominator); - if ( encoder != NULL ) { + if ( encoder ) { + encoder->setContext(packet->context); encoder->setStartTime(record->startTime()); encoder->setTimingQuality(packet->timingQuality); item->encoder = EncoderPtr(encoder); @@ -1012,17 +1233,23 @@ Encoder* Plugin::getEncoder(PacketPtr packet) { } bool Plugin::encodePacket(PacketPtr packet) { - if ( _encoderFactory == NULL || - !_encoderFactory->supportsRecord(packet->record.get()) ) + if ( !_encoderFactory || + !_encoderFactory->supportsRecord(packet->record.get()) ) { return commitPacket(packet); + } Buffer *buffer = packet->record->data(); - if ( buffer == NULL ) return commitPacket(packet); + if ( !buffer ) { + return commitPacket(packet); + } Encoder *encoder = getEncoder(packet); - if ( encoder == NULL ) return commitPacket(packet); + if ( !encoder ) { + return commitPacket(packet); + } - for ( size_t i = 0; i < buffer->size(); i = i + dataTypeSize(packet->dataType) ) { + uint8_t dtSize = dataTypeSize(packet->dataType); + for ( size_t i = 0; i < buffer->size(); i = i + dtSize ) { char *data = buffer->data() + i; encoder->push(data); } @@ -1071,7 +1298,7 @@ bool Plugin::sendPacket(Packet *packet) { // a wrapper function that checks if we can write data or not. // If we do not do the check the system send operation // hangs when the TCP buffer is full and the connection is - // is in a undefined state. + // is in an undefined state. // The check and the system send operation run in a loop until // the data has been sent successfully or the user requests to // to stop => The function call might block. @@ -1079,7 +1306,8 @@ bool Plugin::sendPacket(Packet *packet) { return false; } - CAPS_DEBUG("Packet sent to CAPS, stream: %s, time window: %s~%s", + CAPS_DEBUG("[%p] Packet sent to CAPS, stream: %s, time window: %s~%s", + static_cast(this), packet->streamID.c_str(), packet->record->startTime().iso().c_str(), packet->record->endTime().iso().c_str()); @@ -1098,12 +1326,12 @@ bool Plugin::writeJournal() { //if ( _states.empty() ) return true; if ( !_journalFile.empty() ) { - FS_DECLARE_PATH(path, _journalFile) + fs::path path(_journalFile); string filename = ".journal.tmp"; - if ( FS_HAS_PARENT_PATH(path) ) { - createPath(FS_PARENT_PATH(path).string()); - filename = (FS_PARENT_PATH(path) / FS_PATH(filename)).string(); + if ( path.has_parent_path() ) { + createPath(path.parent_path().string()); + filename = (path.parent_path() / filename).string(); } ofstream ofs(filename.c_str()); @@ -1111,12 +1339,13 @@ bool Plugin::writeJournal() { if ( writeJournal(ofs) ) { ofs.close(); - if ( FS_HAS_PARENT_PATH(path) ) { - createPath(FS_PARENT_PATH(path).string()); + if ( path.has_parent_path() ) { + createPath(path.parent_path().string()); } if ( rename(filename.c_str(), path.string().c_str()) != 0 ) - CAPS_ERROR("Failed to create journal %s, %s(%d)", _journalFile.c_str(), + CAPS_ERROR("[%p] Failed to create journal %s, %s(%d)", + static_cast(this), _journalFile.c_str(), strerror(errno), errno); } else { @@ -1174,23 +1403,24 @@ bool Plugin::send(char *data, int len, int timeout) { return true; } else if ( res == -1 ) { - CAPS_ERROR("Failed to send data: %s. " + CAPS_ERROR("[%p] Failed to send data: %s. " "Forcing reconnect and trying to send data again.", - strerror(errno)); + static_cast(this), strerror(errno)); } else { - CAPS_ERROR("Incomplete send operation: " + CAPS_ERROR("[%p] Incomplete send operation: " "Only %d/%d bytes have been sent. " "Forcing reconnect and trying to send data again.", - len, res); + static_cast(this), len, res); } } else if ( !res ) { - CAPS_ERROR("Detected hanging TCP connection. " - "Forcing reconnect and trying to send data again."); + CAPS_ERROR("[%p] Detected hanging TCP connection. " + "Forcing reconnect and trying to send data again.", + static_cast(this)); } else { - CAPS_ERROR("Send error: %s", strerror(errno)); + CAPS_ERROR("[%p] Send error: %s", static_cast(this), strerror(errno)); } return false; @@ -1198,8 +1428,8 @@ bool Plugin::send(char *data, int len, int timeout) { bool Plugin::setAddress(const string &addr, uint16_t defaultPort) { if ( !_url.parse(addr, defaultPort) ) { - CAPS_ERROR("Failed to set address: %s", - _url.errorString.c_str()); + CAPS_ERROR("[%p] Failed to set address: %s", + static_cast(this), _url.errorString.c_str()); return false; } @@ -1229,17 +1459,13 @@ void Plugin::dumpPackets(bool enable) { _dumpPackets = enable; } -void Plugin::setAgent(const std::string &agent) { - _agent = agent; -} - bool Plugin::getAPIVersion(int &version) { version = 0; int bytesSent = _socket->send(HELLO_REQUEST); if ( bytesSent != strlen(HELLO_REQUEST) ) { - CAPS_ERROR("Could not get CAPS API version: %s", - strerror(errno)); + CAPS_ERROR("[%p] Could not get CAPS API version: %s", + static_cast(this), strerror(errno)); return false; } @@ -1249,8 +1475,8 @@ bool Plugin::getAPIVersion(int &version) { streamsize bytesRead = buf.sgetn(reinterpret_cast(&msgLength), sizeof(int32_t)); if ( bytesRead != sizeof(int32_t) ) { - CAPS_ERROR("Could not get CAPS API version: Expected message length in " - "server response: %s", strerror(errno)); + CAPS_ERROR("[%p] Could not get CAPS API version: Expected message length in " + "server response: %s", static_cast(this), strerror(errno)); return false; } @@ -1277,7 +1503,7 @@ bool Plugin::getAPIVersion(int &version) { string apiStr = line.substr(4); if ( !str2int(version, apiStr.c_str()) ) { - CAPS_ERROR("Invalid CAPS API version: Expected number"); + CAPS_ERROR("[%p] Invalid CAPS API version: Expected number", static_cast(this)); ret = false; } } @@ -1285,5 +1511,187 @@ bool Plugin::getAPIVersion(int &version) { return ret; } +void Plugin::setConnectionID(const std::string &id) { + _connectionID = id; +} + +bool Plugin::setConnectionIDFile(const std::string &filename) { + _connectionIDFile = filename; + if ( fs::exists(_connectionIDFile) ) { + std::ifstream ifs(_connectionIDFile.c_str()); + if ( !ifs.is_open() ) { + CAPS_ERROR("[%p] Could not open connection ID file %s", + static_cast(this), _connectionIDFile.c_str()); + return false; + } + + ifs >> _connectionID; + if ( ifs.fail() ) { + CAPS_ERROR("[%p] Could not read connection ID from file %s", + static_cast(this), _connectionIDFile.c_str()); + return false; + } + } + + return true; +} + +const std::string &Plugin::connectionID() const { + return _connectionID; +} + +bool Plugin::getConnectionID() { + _socket->send("ID "); + _socket->send(_connectionID.data()); + _socket->send("\n"); + + try { + uint32_t size = 0; + auto res = _socket->read(reinterpret_cast(&size), sizeof(size)); + if ( res != sizeof(size) ) { + throw std::runtime_error("Expected connection ID response"); + } + + size = Endianess::Converter::FromLittleEndian(size); + + auto bytesRead = _socket->read(_responseBuf, size); + if ( bytesRead != static_cast(size) ) { + throw std::runtime_error("Server returned not connection ID"); + } + + _responseBuf[size] = '\0'; + + if ( strncmp(_responseBuf, "ID ", 3 ) != 0 ) { + throw std::runtime_error("Invalid connection ID response"); + return false; + } + + _connectionID.assign(_responseBuf + 3, size - 3); + + CAPS_INFO("[%p] CAPS connection ID %s", static_cast(this), + _connectionID.data()); + + if ( !_connectionIDFile.empty() ) { + fs::path path(_connectionIDFile); + + std::string filename = "connection"; + if ( path.has_parent_path() ) { + auto parentPath = path.parent_path().string(); + if ( !createPath(parentPath) ) { + runtime_error("Could not create directory " + parentPath); + } + } + + std::ofstream ofs(_connectionIDFile); + if ( !ofs.is_open() ) { + throw std::runtime_error("Could not create file " + _connectionIDFile); + } + + ofs << _connectionID; + if ( !ofs.good() ) { + throw std::runtime_error("Failed to write connection id to file " + + _connectionIDFile); + } + } + } + catch ( const std::runtime_error &err ) { + CAPS_ERROR("[%p] Connection failed to CAPS at %s:%d: ", + static_cast(this), _url.host.data(), _url.port, + err.what()); + return false; + } + + return true; +} + +void Plugin::setHostInfo(const HostInfo &info) { + _hostInfo = info; +} + +void Plugin::setRuntimeInfo(const RuntimeInfo &info) { + _runtimeInfo = info; + sendRuntimeInfo(); +} + +void Plugin::setConnectedCallback(ConnectedCallback cb) { + _connectedCallback = std::move(cb); +} + +void Plugin::setDisconnectedCallback(DisconnectedCallback cb) { + _disconnectedCallback = std::move(cb); +} + +void Plugin::sendRuntimeInfo() { + if ( !_wasConnected && !_socket ) { + while ( !connect() && !_isExitRequested ) { + disconnect(); + wait(); + } + + if ( !isConnected() ) { + return; + } + } + + std::string buffer; + + if ( _runtimeInfo.cpuUsage != -1 ) { + buffer += "CPU "; + buffer += toString(_runtimeInfo.cpuUsage); + buffer += "\n"; + } + + if ( _runtimeInfo.procUsedMem != -1 ) { + buffer += "MEM "; + buffer += toString(_runtimeInfo.procUsedMem); + buffer += "\n"; + } + + if ( _runtimeInfo.availableDisc != -1 ) { + buffer += "AVAILABLEDISC "; + buffer += toString(_runtimeInfo.availableDisc); + buffer += "\n"; + } + + if ( _runtimeInfo.availableMem != -1 ) { + buffer += "AVAILABLEMEM "; + buffer += toString(_runtimeInfo.availableMem); + buffer += "\n"; + } + + if ( _runtimeInfo.systemLoad != -1 ) { + buffer += "LOAD "; + buffer += toString(_runtimeInfo.systemLoad); + buffer += "\n"; + } + + // Send packet data header and packet header + _socket->setNonBlocking(false); + + // Instead of directly write data to the socket we call + // a wrapper function that checks if we can write data or not. + // If we do not do the check the system send operation + // hangs when the TCP buffer is full and the connection is + // is in an undefined state. + // The check and the system send operation run in a loop until + // the data has been sent successfully or the user requests to + // to stop => The function call might block. + while ( !send(buffer.data(), buffer.size(), _sendTimeout) ) { + CAPS_ERROR("[%p] Sending rti failed: %s", static_cast(this), _isExitRequested ? "abort" : "reconnect"); + readResponse(); + disconnect(); + + while ( !_isExitRequested && !isConnected() ) { + wait(); + disconnect(); + connect(); + } + + if ( !isConnected() ) { + return; + } + } +} + } } diff --git a/libs/gempa/caps/plugin.h b/libs/gempa/caps/plugin.h index 5a8dfc2..7946356 100644 --- a/libs/gempa/caps/plugin.h +++ b/libs/gempa/caps/plugin.h @@ -16,8 +16,9 @@ #ifndef GEMPA_CAPS_PLUGIN_H #define GEMPA_CAPS_PLUGIN_H + /* -// Enable/disable journal +// Enable/disable journal #define CAPS_FEATURES_JOURNAL 1 // Enable/disable backfilling of packets #define CAPS_FEATURES_BACKFILLING 1 @@ -36,7 +37,6 @@ #include #include #include - #include #include @@ -50,10 +50,10 @@ #include - namespace Gempa { namespace CAPS { + class SC_GEMPA_CAPS_API Plugin { public: enum Status { @@ -71,6 +71,22 @@ class SC_GEMPA_CAPS_API Plugin { size_t maxBytesBuffered; }; + struct HostInfo { + std::string agent; + std::string agentVersion; + std::string os; + int64_t totalMem{-1}; + int64_t totalDisc{-1}; + }; + + struct RuntimeInfo { + int64_t availableMem{-1}; + int64_t procUsedMem{-1}; + int64_t availableDisc{-1}; + int cpuUsage{-1}; + double systemLoad{-1}; + }; + typedef std::vector Buffer; typedef boost::shared_ptr BufferPtr; typedef std::deque PacketBuffer; @@ -90,13 +106,15 @@ class SC_GEMPA_CAPS_API Plugin { typedef std::map StreamStates; typedef boost::function PacketAckFunc; + const CAPS::Time &, void *context)> PacketAckFunc; + typedef boost::function ConnectedCallback; + typedef boost::function DisconnectedCallback; public: Plugin(const std::string &name, const std::string &options = "", const std::string &description = ""); - ~Plugin(); + virtual ~Plugin(); void close(); void quit(); @@ -186,15 +204,31 @@ class SC_GEMPA_CAPS_API Plugin { void setPacketAckFunc(const PacketAckFunc &func) { _packetAckFunc = func; } - void setSendTimeout(int timeout) { - _sendTimeout = timeout; + /** + * @brief Sets the connection timout used when the plugin + * tries to establish a connection to CAPS + * @param sec The seconds + */ + void setConnectionTimeout(int sec) { + _connectionTimeout = sec; } - void setTimeouts(int ack, int lastAck) { - _ackTimeout = ack; - _lastAckTimeout = lastAck; + /** + * @brief Sets the maximum number of seconds to wait until + * the file descriptor is ready for writing. + * @param sec The seconds + */ + void setSendTimeout(int sec) { + _sendTimeout = sec; } + /** + * @brief Sets timeout values + * @param ack The acknowledgement message value + * @param lastAck The last acknowledgement message value + * @param send See setSendTimeout + * @ + */ void setTimeouts(int ack, int lastAck, int send) { _ackTimeout = ack; _lastAckTimeout = lastAck; @@ -209,11 +243,18 @@ class SC_GEMPA_CAPS_API Plugin { const StreamStates &streamStates() const { return _states; } bool writeJournal(); bool writeJournal(std::ostream &ostream); + + /** + * @brief Gets end time of the last acknowledged packet + * @param id The stream ID, e.g., GE.APE..BHZ + * @return The last end time or an invalid time + */ + Gempa::CAPS::Time lastEndTime(const std::string &id); #endif Status push(const std::string &net, const std::string &sta, const std::string &loc, const std::string &cha, DataRecordPtr rec, const std::string &uom, - int timingQuality = -1); + int timingQuality = -1, void *context = nullptr); Status push(const std::string &net, const std::string &sta, const std::string &loc, const std::string &cha, @@ -221,10 +262,56 @@ class SC_GEMPA_CAPS_API Plugin { uint16_t numerator, uint16_t denominator, const std::string &uom, void *data, size_t count, - DataType dt, int timingQuality = -1); + DataType dt, int timingQuality = -1, + void *context = nullptr); + + /** + * @brief Sends CAPS data record to CAPS + * @param net Network code + * @param sta Station code + * @param loc Location code + * @param cha Channel code + * @param rec CAPS data record + * @param uom Unit of measurement + * @param timingQuality Data timing quality. Ususally between 0 and 100 or -1 + * if unset + * @param context Pointer to user-defined context data + * @return Status Packet status + */ + Status pushRecord(const std::string &net, const std::string &sta, + const std::string &loc, const std::string &cha, + DataRecordPtr rec, const std::string &uom, + int timingQuality = -1, void *context = nullptr); + + /** + * @brief Sends C array data as RAW record to CAPS + * @param net Network code + * @param sta Station code + * @param loc Location code + * @param cha Channel code + * @param stime Packet start time + * @param numerator Sampling frequency numerator + * @param denominator Sampling frequency denominator + * @param uom Unit of measurement + * @param data Pointer to data array + * @param count Number of samples + * @param dt Data type, e.g., DT_DOUBLE + * @param timingQuality Data timing quality. Ususally between 0 and 100 or -1 + * if unset + * @param context Pointer to user-defined context data + * @return Status Packet status + */ + Status pushRaw(const std::string &net, const std::string &sta, + const std::string &loc, const std::string &cha, + const Time &stime, + uint16_t numerator, uint16_t denominator, + const std::string &uom, + void *data, size_t count, + DataType dt, int timingQuality = -1, + void *context = nullptr); /* - * Sends given data as any packet. Before sending the data will be + * Sends given data as any packet. Before sending the data will be * copied into an internal buffer. We introduced this method * to simplify the creation of the python wrappers. */ @@ -233,13 +320,62 @@ class SC_GEMPA_CAPS_API Plugin { const std::string &loc, const std::string &cha, const Time &stime, uint16_t numerator, uint16_t denominator, const std::string &format, - char *data, size_t count); + char *data, size_t count, void *context = nullptr); Status push(const std::string &net, const std::string &sta, const std::string &loc, const std::string &cha, const Time &stime, uint16_t numerator, uint16_t denominator, const std::string &format, - const std::string &data); + const std::string &data, void *context = nullptr); + + /** + * @brief Sends C array data as any record to CAPS + * @param net Network code + * @param sta Station code + * @param loc Location code + * @param cha Channel code + * @param stime Packet start time + * @param etime Packet end time + * @param numerator Sampling frequency numerator. Use 1 / 0 + * for packets without sampling frequency. + * @param denominator Sampling frequency denominator + * @param format Data format of the provided data + * @param uom Unit of measurement, e.g., px + * @param data Pointer to data array + * @param count Number of data elements + * @param context Pointer to user-defined context data + * @return Status Packet status + */ + Status pushAny(const std::string &net, const std::string &sta, + const std::string &loc, const std::string &cha, + const Time &stime, const Time &etime, + uint16_t numerator, uint16_t denominator, + const std::string &format, const std::string &uom, + char *data, size_t count, void *context = nullptr); + + /** + * @brief Sends STL string as any record to CAPS + * @param net Network code + * @param sta Station code + * @param loc Location code + * @param cha Channel code + * @param stime Packet start time + * @param etime Packet end time + * @param numerator Sampling frequency numerator. Use 1 / 0 + * for packets without sampling frequency. + * @param denominator Sampling frequency denominator + * @param format Data format of the provided data + * @param uom Unit of measurement, e.g., px + * @param data Data as string + * @param context Pointer to user-defined context data + * @return Status Packet status + */ + Status pushAny(const std::string &net, const std::string &sta, + const std::string &loc, const std::string &cha, + const Time &stime, const Time &etime, + uint16_t numerator, uint16_t denominator, + const std::string &format, const std::string &uom, + const std::string &data, void *context = nullptr); #endif void flushEncoders(); @@ -256,12 +392,51 @@ class SC_GEMPA_CAPS_API Plugin { return _packetBuffer; } + /** - * @brief Sets the agent string. Allows the server to - * identify the application that sends data. - * @param agent The agent string + * @brief Set host information like agent or total mem + * @param info The host info struct */ - void setAgent(const std::string &agent); + void setHostInfo(const HostInfo &info); + + /** + * @brief Set runtime information like CPU load or process memory usage. + * This methods blocks until sent. + * @param info The runtime info struct + */ + void setRuntimeInfo(const RuntimeInfo &info); + + /** + * @brief Sets the connection ID. If not set, the server generates an ID + * during the first connection attempt. + * @param id The connection ID string + */ + void setConnectionID(const std::string &id); + + /** + * @brief Set the connection ID file to read the connection ID from + * @param filename The filename + * @return True if the connection could be read from the given file + */ + bool setConnectionIDFile(const std::string &filename); + + /** + * @brief Returns the connection ID + * @return Returns the connection ID + */ + const std::string &connectionID() const; + + /** + * @brief Set function that gets called if the connection to the server + * has been established. + */ + void setConnectedCallback(ConnectedCallback f); + + /** + * @brief Set function that gets called if the connection to the server + * was disconnected. + */ + void setDisconnectedCallback(DisconnectedCallback f); const char *version() { return LIB_CAPS_VERSION_NAME; @@ -277,13 +452,25 @@ class SC_GEMPA_CAPS_API Plugin { */ bool getAPIVersion(int &version); + /** + * @brief Get connection ID from server. If the plugin sends no + * connection ID the server generates one. + * @return True on success + */ + bool getConnectionID(); + + /** + * brief Send runtime information + */ + void sendRuntimeInfo(); + private: typedef boost::shared_ptr EncoderPtr; struct EncoderItem { EncoderItem() : dataType(DT_Unknown) {} - EncoderPtr encoder; - DataType dataType; + EncoderPtr encoder; + DataType dataType; }; typedef std::map EncoderItems; @@ -324,6 +511,8 @@ class SC_GEMPA_CAPS_API Plugin { fd_set _readFDs; fd_set _writeFDs; int _sendTimeout; + int _readTimeout; + int _connectionTimeout; #if !defined(CAPS_FEATURES_JOURNAL) || CAPS_FEATURES_JOURNAL std::string _journalFile; bool _journalDirty; @@ -352,10 +541,17 @@ class SC_GEMPA_CAPS_API Plugin { TimeSpan _maxFutureEndTime; bool _dumpPackets; - std::string _agent; + HostInfo _hostInfo; + RuntimeInfo _runtimeInfo; + std::string _connectionID; + std::string _connectionIDFile; + ConnectedCallback _connectedCallback; + DisconnectedCallback _disconnectedCallback; }; -typedef boost::shared_ptr PluginPtr; + +using PluginPtr = boost::shared_ptr; + } } diff --git a/libs/gempa/caps/pluginapplication.cpp b/libs/gempa/caps/pluginapplication.cpp index a0324f0..6877795 100644 --- a/libs/gempa/caps/pluginapplication.cpp +++ b/libs/gempa/caps/pluginapplication.cpp @@ -20,19 +20,17 @@ #include #include -#include -#include +#include +#include #include -#ifdef SC_GEMPA_SEATTLE -#include -#else -#include -#endif +#include -using namespace std; +#include +#include +namespace fs = std::filesystem; namespace sc = Seiscomp::Core; namespace { @@ -73,52 +71,91 @@ void LogDebug(const char *fmt, ...) { } const size_t MIN_BUFFER_SIZE = 1024*16; -const uint16_t DEFAULT_PORT = 18003; + +bool readKeyValueFile(std::map &data, + const std::string &filename, const char *sep) { + std::ifstream ifs(filename.c_str()); + if ( !ifs.is_open() ) { + return false; + } + + std::string line; + while ( getline(ifs, line) ) { + const char* str = line.c_str(); + int len = strlen(str); + const char *tok = nullptr; + int tok_len = 0; + int tok_count = 0; + + std::string key; + + for ( ; (tok = Gempa::CAPS::tokenize(str, sep, len, tok_len)) != nullptr; + ++tok_count ) { + Gempa::CAPS::trim(tok, tok_len); + if ( tok_count == 0 ) { + if ( len == 0) { + break; + } + + key.assign(tok, tok_len); + } + else if ( tok_count == 1 ) { + data.insert({key, std::string(tok, tok_len)}); + } + } + } + + return true; +} + +int64_t getAvailableMemory() { + std::map data; + if ( !readKeyValueFile(data, "/proc/meminfo", ":") ) { + return -1; + } + + auto it = data.find("MemAvailable"); + if ( it == data.end() ) { + return -1; + } + + std::vector tokens; + sc::split(tokens, it->second.c_str(), "kB"); + if ( tokens.empty() ) { + return -1; + } + + sc::trim(tokens[0]); + int64_t availableMemory = 0; + sc::fromString(availableMemory, tokens[0]); + + return availableMemory; +} + } -namespace Gempa { -namespace CAPS { +namespace Gempa::CAPS { -PluginApplication::PluginApplication(int argc, char **argv, const string &desc) +PluginApplication::PluginApplication(int argc, char **argv, const std::string &name) : Seiscomp::Client::StreamApplication(argc, argv) -, _plugin(Plugin(desc)) { - _bufferSize = 1 << 20; - _backfillingBufferSize = 180; - _flushInterval = 10; - _ackTimeout = 60; - _lastAckTimeout = 5; - _sendTimeout = 60; - _logStatus = false; - _statusFlushInterval = 10; - _strAddr = "localhost:" + sc::toString(DEFAULT_PORT); +, _plugin(Plugin(name)) { - SC_FS_DECLARE_PATH(path, "@ROOTDIR@/var/run/" + SCCoreApp->name() + "/journal") - _journalFile = path.string(); - - _mseedEnabled = false; - _mseedEncoding = Steim2; - _mseedRecordLength = 9; - _strMseedEncoding = "Steim2"; - _maxFutureEndTime = 120; - _dumpPackets = false; + fs::path path("@ROOTDIR@/var/run/" + SCCoreApp->name()); + _journalFile = (path / "journal").string(); // By default we disable the acquisition autostart because not all plugins // require this feature. It must be enabled explicitly if required. setAutoAcquisitionStart(false); + setRecordStreamEnabled(true); } void PluginApplication::createCommandLineDescription() { Seiscomp::Client::StreamApplication::createCommandLineDescription(); commandline().addGroup("Output"); - commandline().addOption("Output", "addr,a", - "Data output address\n" + commandline().addOption("Output", "output,O", + "Data output address. Format:\n" "[[caps|capss]://][user:pass@]host[:port]", &_strAddr); - commandline().addOption("Output", "agent", - "Sets the agent string. Allows " - "the server to identify the " - "application that sends data.", - &_agent); commandline().addOption("Output", "buffer-size,b", "Size (bytes) of the packet buffer", &_bufferSize); commandline().addOption("Output", "backfilling", @@ -133,9 +170,11 @@ void PluginApplication::createCommandLineDescription() { commandline().addOption("Output", "rec-len", "MiniSEED record length expressed as a power of 2." "A 512 byte record would be 9.", &_mseedRecordLength); - commandline().addOption("Output", "max-future-endtime", "Maximum allowed relative end time for packets. If " + commandline().addOption("Output", "max-future-endtime", + "Maximum allowed relative end time for packets. If " "the packet end time is greater than the current time plus this " - "value the packet will be discarded. By default this value is set to 120 seconds.", + "value the packet will be discarded. By default this value is set to 120 seconds. " + "A negative value disables the check.", &_maxFutureEndTime); commandline().addOption("Output", "dump-packets", "Dump packets to stdout"); commandline().addGroup("Journal"); @@ -154,6 +193,14 @@ void PluginApplication::createCommandLineDescription() { commandline().addOption("Status", "status-flush", "Flush status every n " "seconds to disk", &_statusFlushInterval); + + commandline().addGroup("Host"); + commandline().addOption("Host", "host-storage", + "Determine disc capacity and available space from this path", + &_host.storage); + commandline().addOption("Host", "host-os", + "Set host operating system information", + &_host.os); } void PluginApplication::done() { @@ -169,6 +216,8 @@ void PluginApplication::done() { _stats.endTime.valid()?_stats.endTime.iso().c_str():"", _stats.files); + _plugin.close(); + Seiscomp::Client::StreamApplication::done(); } @@ -179,17 +228,29 @@ void PluginApplication::exit(int returnCode) { } void PluginApplication::handleTimeout() { - sc::Time time = sc::Time::GMT().toLocalTime(); + auto time = Time::GMT(); + auto seconds = time.seconds(); - Plugin::Stats stats = _plugin.stats(); - _statusFile.stream() << time.toString("%Y/%m/%d %T") << " " << stats.maxBytesBuffered << endl; + if ( _logStatus && (seconds % _statusFlushInterval == 0) ) { + Plugin::Stats stats = _plugin.stats(); + _statusFile.stream() << time.toLocalTime().toString("%Y/%m/%d %T") << " " + << stats.maxBytesBuffered << std::endl; + _plugin.resetMaxBytesBuffered(); + } + if ( seconds % 3 == 0 ) { + updateRuntimeInfo(); + } - _plugin.resetMaxBytesBuffered(); + if ( seconds % 10 == 0 ) { + sendRuntimeInfo(); + } } bool PluginApplication::init() { - if ( !Seiscomp::Client::StreamApplication::init() ) return false; + if ( !Seiscomp::Client::StreamApplication::init() ) { + return false; + } // Setup log handlers Gempa::CAPS::SetLogHandler(Gempa::CAPS::LL_ERROR, LogError); @@ -198,12 +259,31 @@ bool PluginApplication::init() { Gempa::CAPS::SetLogHandler(Gempa::CAPS::LL_INFO, LogInfo); Gempa::CAPS::SetLogHandler(Gempa::CAPS::LL_DEBUG, LogDebug); + Plugin::HostInfo hostInfo; + if ( !getHostInfo(hostInfo) ) { + return false; + } + _plugin.setBufferSize(_bufferSize); _plugin.setFlushInterval(_flushInterval); + _plugin.setConnectionTimeout(_connectionTimeout); _plugin.setTimeouts(_ackTimeout, _lastAckTimeout, _sendTimeout); _plugin.setMaxFutureEndTime(_maxFutureEndTime); _plugin.dumpPackets(_dumpPackets); - _plugin.setAgent(_agent); + _plugin.setHostInfo(hostInfo); + + LogInfo("CAPS connection settings\n" + " Output CAPS server : %s\n" + " Buffer size : %zu bytes\n" + " Backfilling buffer size : %zu s\n" + " Max future end time : %d s\n" + " Connection timeout : %d s\n" + " Timeouts Ack/LastAck/Send : %d s/%d s/%d s\n" + " Agent : %s\n" + " Agent version : %s", + _strAddr.c_str(), _bufferSize, _backfillingBufferSize, _maxFutureEndTime, + _connectionTimeout, _ackTimeout, _lastAckTimeout, _sendTimeout, + hostInfo.agent.data(), hostInfo.agentVersion.data()); if ( _mseedEnabled ) { MSEEDEncoderFactory *factory = nullptr; @@ -238,7 +318,13 @@ bool PluginApplication::init() { if ( _backfillingBufferSize > 0 ) { _plugin.setBackfillingBufferSize(_backfillingBufferSize); - LogInfo("Backfilling buffer size set to %d", _backfillingBufferSize); + } + + std::string connectionIDFile = "@ROOTDIR@/var/run/" + name() + "/id"; + connectionIDFile = Seiscomp::Environment::Instance()->absolutePath(connectionIDFile); + LogInfo("Reading connection ID from %s", connectionIDFile.c_str()); + if ( !_plugin.setConnectionIDFile(connectionIDFile) ) { + return false; } if ( !_journalFile.empty() ) { @@ -251,20 +337,25 @@ bool PluginApplication::init() { } if ( _logStatus ) { - string filename = Seiscomp::Environment::Instance()->logDir() + "/" + SCCoreApp->name() + "-stats.log"; + std::string filename = Seiscomp::Environment::Instance()->logDir() + "/" + SCCoreApp->name() + "-stats.log"; if ( !_statusFile.open(filename.c_str()) ) { LogError("Could not open status file %s.", filename.c_str()); return false; } - - enableTimer(_statusFlushInterval); } + // This causes a connect + updateRuntimeInfo(); + _plugin.setRuntimeInfo(_runtimeInfo); + enableTimer(1); + return true; } bool PluginApplication::initConfiguration() { - if ( !Seiscomp::Client::StreamApplication::initConfiguration() ) return false; + if ( !Seiscomp::Client::StreamApplication::initConfiguration() ) { + return false; + } try { _plugin.setHost(configGetString("output.host")); @@ -280,8 +371,11 @@ bool PluginApplication::initConfiguration() { try { _sendTimeout = configGetInt("output.timeout"); } catch ( ... ) { } + try { _connectionTimeout = configGetInt("output.connectionTimeout"); } + catch ( ... ) { } + try { - string addr = configGetString("output.address"); + std::string addr = configGetString("output.address"); if ( !_plugin.setAddress(addr) ) { return false; } @@ -305,8 +399,10 @@ bool PluginApplication::initConfiguration() { catch ( ... ) {} try { - string str = configGetString("output.mseed.encoding"); - if ( !fromString(_mseedEncoding, str)) return false; + std::string str = configGetString("output.mseed.encoding"); + if ( !fromString(_mseedEncoding, str)) { + return false; + } } catch ( ... ) {} @@ -319,16 +415,6 @@ bool PluginApplication::initConfiguration() { try { _maxFutureEndTime = configGetInt("output.maxFutureEndTime"); } catch ( ... ) { } - try { _agent = configGetString("output.agent"); } - catch ( ... ) { - if ( version() ) { - _agent = name() + string(" ") + version(); - } - else { - _agent = name(); - } - } - try { _journalFile = configGetString("journal.file"); } catch ( ... ) {} @@ -341,6 +427,14 @@ bool PluginApplication::initConfiguration() { try { _lastAckTimeout = configGetInt("journal.waitForLastAck"); } catch ( ... ) { } + _host.agent = name(); + try { _host.storage = configGetString("host.storage"); } + catch ( ... ) { } + + try { _host.os = configGetString("host.os"); } + catch ( ... ) { } + + try { _logStatus = configGetBool("statusLog.enable"); } catch ( ... ) { } @@ -351,7 +445,9 @@ bool PluginApplication::initConfiguration() { } bool PluginApplication::validateParameters() { - if ( !Seiscomp::Client::StreamApplication::validateParameters() ) return false; + if ( !Seiscomp::Client::StreamApplication::validateParameters() ) { + return false; + } if ( commandline().hasOption("mseed") ) { _mseedEnabled = true; @@ -366,7 +462,9 @@ bool PluginApplication::validateParameters() { } if ( commandline().hasOption("encoding") ) { - if ( !fromString(_mseedEncoding, _strMseedEncoding)) return false; + if ( !fromString(_mseedEncoding, _strMseedEncoding)) { + return false; + } } if ( _bufferSize < MIN_BUFFER_SIZE ) { @@ -375,23 +473,28 @@ bool PluginApplication::validateParameters() { return false; } - if ( commandline().hasOption("addr") ) { - if ( !_plugin.setAddress(_strAddr, DEFAULT_PORT) ) { + if ( commandline().hasOption("output") ) { + if ( !_plugin.setAddress(_strAddr, 18003) ) { return false; } } + _host.storage = Seiscomp::Environment::Instance()->absolutePath(_host.storage); + return true; } -bool PluginApplication::fromString(MseedEncoding &enc, string str) { +bool PluginApplication::fromString(MseedEncoding &enc, std::string str) { boost::to_lower(str); - if( str == "uncompressed" ) + if( str == "uncompressed" ) { enc = Uncompressed; - else if ( str == "steim1" ) + } + else if ( str == "steim1" ) { enc = Steim1; - else if ( str == "steim2" ) + } + else if ( str == "steim2" ) { enc = Steim2; + } else { SEISCOMP_ERROR("Unsupported encoding %s", str.c_str()); return false; @@ -400,5 +503,60 @@ bool PluginApplication::fromString(MseedEncoding &enc, string str) { return true; } +bool PluginApplication::getHostInfo(Plugin::HostInfo &hostInfo) { + hostInfo.agent = _host.agent; + hostInfo.agentVersion = version() ? version() : ""; + hostInfo.totalMem = _hostInfo.totalMemory(); + + try { + auto spaceInfo = fs::space(_host.storage); + hostInfo.totalDisc = spaceInfo.capacity / 1024; + } + catch ( const fs::filesystem_error& e ) { + SEISCOMP_ERROR("Failed to determine filesystem information: %s", e.what()); + return false; + } + + hostInfo.os = _host.os; + if ( hostInfo.os.empty() ) { + std::map osRelease; + if ( !readKeyValueFile(osRelease, "/etc/os-release", "=") ) { + SEISCOMP_WARNING("Unable to read file /etc/os-release"); + } + + auto it = osRelease.find("PRETTY_NAME"); + if ( it != osRelease.end() ) { + auto unquote = [](const std::string& str) { + if ( str.length() >= 2 && str.front() == '"' && str.back() == '"' ) { + return str.substr(1, str.length() - 2); + } + return str; + }; + + hostInfo.os = unquote(it->second); + } + } + + return true; } + +void PluginApplication::updateRuntimeInfo() { + _runtimeInfo.cpuUsage = std::max(0, static_cast(_hostInfo.getCurrentCpuUsage() * 1000)); + _runtimeInfo.procUsedMem = _hostInfo.getCurrentMemoryUsage(); + _runtimeInfo.availableMem = getAvailableMemory(); + getloadavg(&_runtimeInfo.systemLoad, 1); + + try { + auto spaceInfo = fs::space(_host.storage); + _runtimeInfo.availableDisc = spaceInfo.available / 1024; + } + catch ( const fs::filesystem_error& e ) { + SEISCOMP_WARNING("Failed to determine filesystem information: %s", e.what()); + } +} + +void PluginApplication::sendRuntimeInfo() { + _plugin.setRuntimeInfo(_runtimeInfo); +} + } diff --git a/libs/gempa/caps/pluginapplication.h b/libs/gempa/caps/pluginapplication.h index 83ff19d..9867bb4 100644 --- a/libs/gempa/caps/pluginapplication.h +++ b/libs/gempa/caps/pluginapplication.h @@ -16,98 +16,153 @@ #ifndef GEMPA_CAPS_PLUGINAPPLICATION_H #define GEMPA_CAPS_PLUGINAPPLICATION_H + #include #include -#include -#include -#include +#include +#include +#if SC_API_VERSION < SC_API_VERSION_CHECK(17, 0, 0) +#include +#else +#include +#endif +#include +#include + + +namespace Gempa::CAPS { -namespace Gempa { -namespace CAPS { class SC_GEMPA_CAPS_API PluginApplication : public Seiscomp::Client::StreamApplication { public: PluginApplication(int argc, char **argv, - const std::string &desc = ""); + const std::string &name = ""); protected: + struct Statistics { + Statistics() = default; + + uint32_t records{0}; + uint32_t samples{0}; + Time startTime; + Time endTime; + uint32_t gaps{0}; + uint32_t files{0}; + }; + + /** + * @brief The MseedEncoding enum defines supported MSEED encodings. + */ + enum MseedEncoding {Uncompressed, Steim1, Steim2}; + /** * @brief Adds common plugin commandline options. */ - virtual void createCommandLineDescription(); + void createCommandLineDescription() override; - virtual void done(); + /** + * @brief Cleanup method called before exec() returns. + */ + void done() override; - virtual void exit(int returnCode); + /** + * @brief This method is called before the applications exits. + * @param returnCode The exit code + */ + void exit(int returnCode) override; - virtual void handleRecord(Seiscomp::Record *record) {} - virtual void handleTimeout(); + /** + * @brief Handles records. This is dummy implemenation as the base + * class requires to implement the method. + * @param record Pointer to record + */ + void handleRecord(Seiscomp::Record *record) override {} + + /** + * @brief Handles timer events. For instance writes the plugin states + * information to file. + */ + void handleTimeout() override; /** * @brief Initialization method. */ - virtual bool init(); + bool init() override; /** * @brief Reads common plugin configuration options. */ - virtual bool initConfiguration(); + bool initConfiguration() override; /** * @brief Validates command line parameters. */ - virtual bool validateParameters(); + bool validateParameters() override; - protected: - struct Statistics { - uint32_t records; - uint32_t samples; - Time startTime; - Time endTime; - uint32_t gaps; - uint32_t files; - - Statistics() : records(0), samples(0), gaps(0), files(0) {} - }; - - enum MseedEncoding {Uncompressed, Steim1, Steim2}; - - bool fromString(MseedEncoding &enc, std::string str); - - protected: class FileRotator : public Seiscomp::Logging::FileRotatorOutput { public: std::ofstream& stream() { return _stream; } }; + /** + * @brief Get MSEED encoding value from string + * @param enc The encoding + * @param str The encoding as string + * @return True on success + */ + static bool fromString(MseedEncoding &enc, std::string str); + + /** + * @brief Get host information like total mem or OS + * @param hostInfo Host info struct + * @return True on success + */ + bool getHostInfo(Plugin::HostInfo &hostInfo); + + /** + * @brief Get runtime information + */ + void updateRuntimeInfo(); + + virtual void sendRuntimeInfo(); + protected: - Plugin _plugin; - std::string _strAddr; - size_t _bufferSize; - size_t _backfillingBufferSize; - bool _mseed; - std::string _journalFile; - int _flushInterval; - int _ackTimeout; - int _lastAckTimeout; - int _sendTimeout; - int _maxFutureEndTime; - Statistics _stats; - bool _mseedEnabled; - MseedEncoding _mseedEncoding; - uint _mseedRecordLength; - std::string _strMseedEncoding; - std::string _strPacketType; - Seiscomp::Util::Timer _timer; - FileRotator _statusFile; - bool _logStatus; - uint _statusFlushInterval; - bool _dumpPackets; - std::string _agent; + struct Host { + std::string agent; + std::string storage{"@LOGDIR@"}; + std::string os; + }; + + Plugin _plugin; + std::string _strAddr{"localhost:18003"}; + size_t _bufferSize{1 << 20}; + size_t _backfillingBufferSize{180}; + bool _mseed; + std::string _journalFile; + int _flushInterval{10}; + int _ackTimeout{60}; + int _lastAckTimeout{5}; + int _sendTimeout{60}; + int _connectionTimeout{30}; + int _maxFutureEndTime{120}; + Statistics _stats; + bool _mseedEnabled{false}; + MseedEncoding _mseedEncoding{Steim2}; + uint _mseedRecordLength{9}; + std::string _strMseedEncoding{"Steim2"}; + std::string _strPacketType; + Seiscomp::Util::Timer _timer; + FileRotator _statusFile; + bool _logStatus{false}; + uint _statusFlushInterval{10}; + bool _dumpPackets{false}; + Seiscomp::System::HostInfo _hostInfo; + Host _host; + Plugin::RuntimeInfo _runtimeInfo; }; -} + } diff --git a/libs/gempa/caps/pluginpacket.h b/libs/gempa/caps/pluginpacket.h index 9b6604c..e193977 100644 --- a/libs/gempa/caps/pluginpacket.h +++ b/libs/gempa/caps/pluginpacket.h @@ -16,18 +16,28 @@ #ifndef GEMPA_CAPS_PLUGINPACKET_H #define GEMPA_CAPS_PLUGINPACKET_H -#include "mseed/encoder.h" + +#include "packet.h" + namespace Gempa { namespace CAPS { + struct Packet { Packet() : timingQuality(-1) {} Packet(DataRecordPtr rec, const std::string &networkCode, const std::string &stationCode, - const std::string &locationCode, const std::string &channelCode) - : record(rec), networkCode(networkCode), stationCode(stationCode), - locationCode(locationCode), channelCode(channelCode), timingQuality(-1) { + const std::string &locationCode, const std::string &channelCode, + void *context = nullptr) + : record(rec) + , networkCode(networkCode) + , stationCode(stationCode) + , locationCode(locationCode) + , channelCode(channelCode) + , timingQuality(-1) + , context(context) + { streamID = networkCode + "." + stationCode + "." + locationCode + "." + channelCode; } @@ -42,28 +52,33 @@ struct Packet { return packet; } - typedef std::vector Buffer; - typedef boost::shared_ptr BufferPtr; + using Buffer = std::vector; + using BufferPtr = boost::shared_ptr; - BufferPtr buffer; // PacketDataHeader, PacketHeader[V1|V2], Data - DataRecordPtr record; - std::string networkCode; - std::string stationCode; - std::string locationCode; - std::string channelCode; - std::string streamID; - DataType dataType; + BufferPtr buffer; // PacketDataHeader, PacketHeader[V1|V2], Data + DataRecordPtr record; + std::string networkCode; + std::string stationCode; + std::string locationCode; + std::string channelCode; + std::string streamID; + DataType dataType; #if !defined(CAPS_FEATURES_BACKFILLING) || CAPS_FEATURES_BACKFILLING - int64_t dt_us; // Length of one sample in microseconds + int64_t dt_us; // Length of one sample in microseconds #endif - std::string uom; - int timingQuality; + std::string uom; + int timingQuality; + void *context; + size_t size() const { return buffer->size(); } }; -typedef boost::shared_ptr PacketPtr; + +using PacketPtr = boost::shared_ptr; + } } + #endif diff --git a/libs/gempa/caps/rawpacket.cpp b/libs/gempa/caps/rawpacket.cpp index a326126..2094e5b 100644 --- a/libs/gempa/caps/rawpacket.cpp +++ b/libs/gempa/caps/rawpacket.cpp @@ -83,6 +83,10 @@ inline Time getEndTime(const Time &stime, size_t count, const DataRecord::Header RawDataRecord::RawDataRecord() : _dataOfs(0), _dataSize(0), _dirty(false) {} +DataRecord *RawDataRecord::clone() const { + return new RawDataRecord(*this); +} + const char *RawDataRecord::formatName() const { DT2STR("RAW/", _currentHeader.dataType) } @@ -258,11 +262,13 @@ DataRecord::ReadStatus RawDataRecord::getData(streambuf &buf, int size, if ( (start.valid() || end.valid()) && _endTime.valid() ) { // Out of bounds? - if ( end.valid() && (end <= _startTime) ) + if ( end.valid() && (end <= _startTime) ) { return RS_AfterTimeWindow; + } - if ( start.valid() && (start >= _endTime) ) + if ( start.valid() && (start >= _endTime) ) { return RS_BeforeTimeWindow; + } // Trim packet front if ( _startTime < start ) { @@ -284,7 +290,7 @@ DataRecord::ReadStatus RawDataRecord::getData(streambuf &buf, int size, // return a partial record if ( maxSamples < sampleCount ) { CAPS_DEBUG("Clip %d available samples to %d", sampleCount, maxSamples); - _endTime -= samplesToTimeSpan(_header, sampleCount-maxSamples); + _endTime -= samplesToTimeSpan(_header, sampleCount - maxSamples); sampleCount = maxSamples; partial = true; } @@ -311,37 +317,45 @@ DataRecord::ReadStatus RawDataRecord::getData(streambuf &buf, int size, switch ( _header.dataType ) { case DT_INT8: - { - // Stay with little endian data - RIFF::VectorChunk<1,false> dataChunk(_data, sampleOfs, sampleCount); - if ( !dataChunk.get(buf, size) ) return RS_Error; - } + { + // Stay with little endian data + RIFF::VectorChunk<1,false> dataChunk(_data, sampleOfs, sampleCount); + if ( !dataChunk.get(buf, size) ) { + return RS_Error; + } + } break; case DT_INT16: - { - // Stay with little endian data - RIFF::VectorChunk<2,false> dataChunk(_data, sampleOfs, sampleCount); - if ( !dataChunk.get(buf, size) ) return RS_Error; - } + { + // Stay with little endian data + RIFF::VectorChunk<2,false> dataChunk(_data, sampleOfs, sampleCount); + if ( !dataChunk.get(buf, size) ) { + return RS_Error; + } + } break; case DT_INT32: case DT_FLOAT: - { - // Stay with little endian data - RIFF::VectorChunk<4,false> dataChunk(_data, sampleOfs, sampleCount); - if ( !dataChunk.get(buf, size) ) return RS_Error; - } + { + // Stay with little endian data + RIFF::VectorChunk<4,false> dataChunk(_data, sampleOfs, sampleCount); + if ( !dataChunk.get(buf, size) ) { + return RS_Error; + } + } break; case DT_INT64: case DT_DOUBLE: - { - // Stay with little endian data - RIFF::VectorChunk<8,false> dataChunk(_data, sampleOfs, sampleCount); - if ( !dataChunk.get(buf, size) ) return RS_Error; - } + { + // Stay with little endian data + RIFF::VectorChunk<8,false> dataChunk(_data, sampleOfs, sampleCount); + if ( !dataChunk.get(buf, size) ) { + return RS_Error; + } + } break; default: @@ -354,7 +368,7 @@ DataRecord::ReadStatus RawDataRecord::getData(streambuf &buf, int size, _dataOfs = 0; _dataSize = _data.size(); - return partial?RS_Partial:RS_Complete; + return partial ? RS_Partial : RS_Complete; } bool RawDataRecord::put(std::streambuf &buf, bool withHeader) const { diff --git a/libs/gempa/caps/rawpacket.h b/libs/gempa/caps/rawpacket.h index 7066c96..77c4d67 100644 --- a/libs/gempa/caps/rawpacket.h +++ b/libs/gempa/caps/rawpacket.h @@ -18,12 +18,12 @@ #include -#include namespace Gempa { namespace CAPS { + struct SC_GEMPA_CAPS_API RawResponseHeader { int64_t timeSeconds; int32_t timeMicroSeconds; @@ -47,7 +47,8 @@ class RawDataRecord : public DataRecord { public: RawDataRecord(); - const char *formatName() const; + DataRecord *clone() const override; + const char *formatName() const override; /** * @brief Reads metadata from data record header @@ -59,7 +60,7 @@ class RawDataRecord : public DataRecord { */ bool readMetaData(std::streambuf &buf, int size, Header &header, - Time &startTime, Time &endTime); + Time &startTime, Time &endTime) override; void setHeader(const Header &header); @@ -67,33 +68,33 @@ class RawDataRecord : public DataRecord { * @brief Returns the meta information of the data if supported * @return The data record header */ - const Header *header() const; + const Header *header() const override; /** * @brief Returns the start time of the record * @return The start time */ - Time startTime() const; + Time startTime() const override; /** * @brief Returns the end time of the record * @return The end time */ - Time endTime() const; + Time endTime() const override; /** * @brief canTrim checks if data trimming is possible * without modifying preceding data * @return True, if data trimming is possible */ - bool canTrim() const; + bool canTrim() const override; /** * @brief canMerge checks if data merging is possible * without modifying preceding data * @return True, if data merging is possible */ - bool canMerge() const; + bool canMerge() const override; /** * @brief Trims a record to start and end time. Trimming @@ -109,7 +110,7 @@ class RawDataRecord : public DataRecord { * @return It returns true if trimming has been done or false * if an error occured or trimming is not supported. */ - bool trim(const Time &start, const Time &end) const; + bool trim(const Time &start, const Time &end) const override; /** * @brief Returns the data size in bytes if the current state would @@ -118,7 +119,7 @@ class RawDataRecord : public DataRecord { * @param withHeader Take header into account * @return Returns the data size in bytes */ - size_t dataSize(bool withHeader) const; + size_t dataSize(bool withHeader) const override; /** * @brief Reads the packet data including header from a streambuf @@ -137,7 +138,7 @@ class RawDataRecord : public DataRecord { ReadStatus get(std::streambuf &buf, int size, const Time &start = Time(), const Time &end = Time(), - int maxBytes = -1); + int maxBytes = -1) override; /** * @brief Reads the packet data without header from a streambuf @@ -167,13 +168,13 @@ class RawDataRecord : public DataRecord { * @param Take header into account * @return True, if the data has been written */ - bool put(std::streambuf &buf, bool withHeader) const; + bool put(std::streambuf &buf, bool withHeader) const override; /** * @brief Returns the packet type * @return The packet type */ - PacketType packetType() const { return RawDataPacket; } + PacketType packetType() const override { return RawDataPacket; } /** * @brief Sets the start time of the record @@ -218,18 +219,19 @@ class RawDataRecord : public DataRecord { class FixedRawDataRecord : public RawDataRecord { public: - virtual bool canTrim() const { return false; } - virtual bool canMerge() const { return false; } - virtual bool trim(const Time &start, const Time &end) const { return false; } - virtual const char *formatName() const; - virtual ReadStatus get(std::streambuf &buf, int size, + bool canTrim() const override { return false; } + bool canMerge() const override { return false; } + bool trim(const Time &start, const Time &end) const override { return false; } + const char *formatName() const override; + ReadStatus get(std::streambuf &buf, int size, const Time &/*start*/, const Time &/*end*/, - int maxBytes) { + int maxBytes) override { return RawDataRecord::get(buf, size, Time(), Time(), maxBytes); } - PacketType packetType() const { return FixedRawDataPacket; } + PacketType packetType() const override { return FixedRawDataPacket; } }; + } } diff --git a/libs/gempa/caps/recordbuilder.h b/libs/gempa/caps/recordbuilder.h index 9569e40..d4ae1da 100644 --- a/libs/gempa/caps/recordbuilder.h +++ b/libs/gempa/caps/recordbuilder.h @@ -16,15 +16,18 @@ #ifndef GEMPA_CAPS_RECORD_SAMPLER_H #define GEMPA_CAPS_RECORD_SAMPLER_H -#include +#include #include +#include #include + namespace Gempa { namespace CAPS { + template class RecordBuilder { public: struct Record { @@ -40,7 +43,8 @@ template class RecordBuilder { void *userData; }; typedef boost::shared_ptr RecordPtr; - typedef boost::function FlushCallback; + typedef std::function FlushCallback; + public: RecordBuilder() : _bufSize(64) {} @@ -130,6 +134,7 @@ template class RecordBuilder { void *_userData; }; + } } diff --git a/libs/gempa/caps/rtcm2packet.cpp b/libs/gempa/caps/rtcm2packet.cpp index 91485af..e510ffb 100644 --- a/libs/gempa/caps/rtcm2packet.cpp +++ b/libs/gempa/caps/rtcm2packet.cpp @@ -61,6 +61,11 @@ void RTCM2DataRecord::setSamplingFrequency(uint16_t numerator, uint16_t denomina } +DataRecord *RTCM2DataRecord::clone() const { + return new RTCM2DataRecord(*this); +} + + const char *RTCM2DataRecord::formatName() const { return "RTC2"; } diff --git a/libs/gempa/caps/rtcm2packet.h b/libs/gempa/caps/rtcm2packet.h index 856a2fe..c228983 100644 --- a/libs/gempa/caps/rtcm2packet.h +++ b/libs/gempa/caps/rtcm2packet.h @@ -20,11 +20,11 @@ #include #include -#include namespace Gempa { namespace CAPS { + class RTCM2DataRecord : public DataRecord { public: struct RTCM2Header : Header { @@ -61,37 +61,38 @@ class RTCM2DataRecord : public DataRecord { RTCM2DataRecord(); - const char* formatName() const; + DataRecord *clone() const override; + const char* formatName() const override; void setTimeStamp(const Time &ts); void setSamplingFrequency(uint16_t numerator, uint16_t denominator); - virtual bool readMetaData(std::streambuf &buf, int size, - Header &header, - Time &startTime, - Time &endTime); + bool readMetaData(std::streambuf &buf, int size, + Header &header, + Time &startTime, + Time &endTime) override; - virtual const Header *header() const; - virtual Time startTime() const; - virtual Time endTime() const; + const Header *header() const override; + Time startTime() const override; + Time endTime() const override; - virtual bool canTrim() const; - virtual bool canMerge() const; + bool canTrim() const override; + bool canMerge() const override; - virtual bool trim(const Time &start, - const Time &end) const; + bool trim(const Time &start, + const Time &end) const override; - virtual size_t dataSize(bool withHeader) const; + size_t dataSize(bool withHeader) const override; - virtual ReadStatus get(std::streambuf &buf, int size, - const Time &start, - const Time &end, - int maxSize = -1); + ReadStatus get(std::streambuf &buf, int size, + const Time &start, + const Time &end, + int maxSize = -1) override; - virtual bool put(std::streambuf &buf, bool withHeader) const; + bool put(std::streambuf &buf, bool withHeader) const override; - PacketType packetType() const { return RTCM2Packet; } + PacketType packetType() const override { return RTCM2Packet; } private: diff --git a/libs/gempa/caps/sessiontable.h b/libs/gempa/caps/sessiontable.h index 22c5e80..7bbf9a6 100644 --- a/libs/gempa/caps/sessiontable.h +++ b/libs/gempa/caps/sessiontable.h @@ -16,9 +16,11 @@ #ifndef GEMPA_CAPS_SESSIONTABLE_H #define GEMPA_CAPS_SESSIONTABLE_H + #include "packet.h" #include +#include #include #include @@ -26,6 +28,7 @@ namespace Gempa { namespace CAPS { + struct SessionTableItem { SessionTableItem() : samplingFrequency(0), samplingFrequencyDivider(0), fSamplingFrequency(0.0), dataType(DT_Unknown), @@ -42,8 +45,8 @@ struct SessionTableItem { DataType dataType; int dataSize; UOM uom; - Time startTime; - Time endTime; + std::optional