You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

527 lines
18 KiB
Plaintext

#!/usr/bin/env seiscomp-python
from __future__ import print_function
import os
import sys
from datetime import datetime
from nettab.convertUtils import StationAttributes, NetworkAttributes, StationMappings, parseDate, formatDate, quote, hummanStr
from nettab.tab import Tab
from optparse import OptionParser
from nettab.nodesi import Instruments
class TabConverter:
def __init__(self, networkCode):
self.__fmt__ = None
self.takeSugestions = None
self.filename = None
self.networkCode = networkCode
self.stationList = None
self.nat = None
self.sat = None
self.sma = None
self.inst = None
self.defaultEpoch = parseDate("1980/001")
self.start=0
self.code=0
self.description=0
self.datalogger=0
self.sensor=0
self.channel=0
self.gaind = 0
self.longitude=0
self.latitude=0
self.elevation=0
self.end=0
self.depth=0
self.orientation=0
## default dates
self.startDate = parseDate("1980/001")
self.endDate = parseDate(None)
def loadStationMapping(self, filename):
if self.networkCode is None: raise Exception("Cannot load Station mapping without network code")
if self.stationList is None: raise Exception("Cannot load Station mapping without station list")
try:
sm = StationMappings(self.networkCode, self.stationList, filename)
self.sma = sm
except Exception as e:
raise e
def loadStationAttribute(self, filename):
if self.networkCode is None: raise Exception("Cannot load Station att without network code")
if self.stationList is None: raise Exception("Cannot load Station att without station list")
try:
sa = StationAttributes(self.networkCode, self.stationList, filename)
self.sat = sa
except Exception as e:
raise e
def loadNetworkAttribute(self, filename):
if self.networkCode is None: raise Exception("Cannot load Network att without network code")
if self.stationList is None: raise Exception("Cannot load Network att without station list")
try:
na = NetworkAttributes(self.networkCode, filename)
self.nat = na
except Exception as e:
raise e
def loadInstrumentsFile(self, filename, filterFolder):
tab = Tab(filterFolder=filterFolder)
tab.digest(filename)
if tab.i:
self.inst = tab.i
def __fmtline__(self):
if not self.__fmt__:
fmt = "Sl: "
fmt += "%%-%ds" % self.code
fmt += " %%-%ds" % self.description
fmt += " %%-%ds" % self.datalogger
fmt += " %%-%ds" % self.sensor
fmt += " %%-%ds" % self.channel
fmt += " %%-%ds" % self.orientation
fmt += " %%-%ds" % self.latitude
fmt += " %%-%ds" % self.longitude
fmt += " %%-%ds" % self.elevation
fmt += " %%-%ds" % self.depth
fmt += " %%-%ds" % self.start
fmt += " %%-%ds" % self.end
self.__fmt__ = fmt
return self.__fmt__
def __analyseLine__(self, items):
inputLine = " ".join(items)
if len(items) < 4:
raise Exception("Invalid items count on line %s" % inputLine)
if len(items) <= 5:
netCode = items[2]
if netCode != self.networkCode:
raise Exception("Tab file (%s) doesn't match class (%s) -- %s" % (netCode,self.networkCode,inputLine))
return [None, None, None]
else:
if len(items) < 6:
raise Exception("Invalid Station line %s" % inputLine)
stationCode = items.pop(0)
code = len(stationCode)
self.code=max(self.code,code)
description = len(quote(hummanStr(items.pop(0))))
self.description=max(self.description, description)
datalogger = len(items.pop(0))
self.datalogger=max(self.datalogger, datalogger)
sensor = len(items.pop(0))
self.sensor=max(self.sensor, sensor)
# Gain
gaind = items.pop(0)
if float(gaind) != 1.0:
self.datalogger = max (self.datalogger, datalogger + len(gaind))
channel = len(items.pop(0))
self.channel=max(self.channel, channel)
latitude = len(items.pop(0))
self.latitude=max(self.latitude, latitude)
longitude = len(items.pop(0))
self.longitude=max(self.longitude, longitude)
elevation = len(items.pop(0))
self.elevation=max(self.elevation, elevation)
#Orientation
depth = items.pop(0)
try:
float(depth)
orientation="ZNE"
except:
orientation = "Z"
(depth,a1,a2) = depth.split("/")
a1n = float(a1)
a2n = float(a2)
orientation+="1"
if a1n != 0.0: orientation += "(0.0,%s)"%a1
orientation+="2"
if a2n != 90.0: orientation+="(0.0,%s)"%a1
orientation = len(orientation)
self.orientation=max(self.orientation, orientation)
depth = len(depth)
self.depth=max(self.depth, depth)
# Start
try:
start = parseDate(items.pop(0))
self.start = max (self.start, len(formatDate(start)))
except:
raise Exception ("Invalid Station line start date %s" % inputLine)
# End
try:
end = parseDate(items.pop(0))
except:
end=parseDate("")
pass
self.end = max (self.end, len(formatDate(end)))
return [stationCode, start, end]
def preload(self, filename, takeSugestions):
self.takeSugestions = takeSugestions
sugestedStart = datetime.now()
sugestedEnd = self.defaultEpoch
stationList = []
error = []
# Some initialization
if self.filename is not None:
raise Exception("Cannot pre-load two different files (current one is %s)" % self.filename)
print("Analysing ... ", file=sys.stderr)
fd = open(filename)
for line in fd:
line = line.strip()
if not line or line[0] == "#": continue
try:
(stationCode, start, end) = self.__analyseLine__(line.split())
except Exception as e:
error.append(str(e))
continue
if not stationCode: continue
if stationCode not in stationList:
stationList.append(stationCode)
sugestedStart = min(sugestedStart, start)
if end and sugestedEnd:
sugestedEnd = max(sugestedEnd, end)
else:
sugestedEnd = None
fd.close()
if len(error):
raise Exception("\n".join(error))
print(" Loaded %d different stations" % len(stationList), file=sys.stderr)
if takeSugestions:
print(" Taking suggestion start date of %s " % formatDate(self.startDate), file=sys.stderr)
self.startDate = sugestedStart
print(" Taking suggestion end date of %s " % formatDate(self.endDate), file=sys.stderr)
self.endDate = sugestedEnd
self.filename = filename
self.stationList = stationList
print("Done.", file=sys.stderr)
def __convertHeader__(self, line, fdo):
# Split line
items = line.split()
if not self.takeSugestions:
if self.nat.hasStart:
print(" Using start from attribute.", file=sys.stderr)
self.startDate = self.nat.startDate
if self.nat.hasEnd:
print(" Using end from attribute.", file=sys.stderr)
self.endDate = self.nat.endDate
nCode = items[2].strip()
if nCode != self.networkCode:
raise Exception("Wrong network code found: %s != %s" % (self.networkCode, nCode))
fdo.write("Nw: %s %s %s" % (nCode, formatDate(self.startDate), formatDate(self.endDate)) + "\n")
self.nat.dump(fdo)
def __convertLine__(self, line, fdo, atFront):
lnfmt = self.__fmtline__()
# Split line
items = line.split()
try:
code = items.pop(0)
except Exception as e:
raise Exception ("Missing Code on %s" % line)
if code not in self.stationList:
raise Exception("Unknow station code $s" % code)
try:
hummanStr(items.pop(0))
except Exception as e:
raise Exception ("Missing Gain on %s" % line)
try:
datalogger = items.pop(0)
except Exception as e:
raise Exception ("Missing Datalogger on %s" % line)
try:
sensor = items.pop(0)
except Exception as e:
raise Exception ("Missing Sensor on %s" % line)
try:
gaind = items.pop(0)
if float(gaind) != 1.0:
if not self.inst:
raise Exception("Instrument database needed to convert gain")
try:
dte = self.inst.dls[str(datalogger).split("%")[0]]
except Exception as e:
print(e, file=sys.stderr)
raise Exception("Datalogger %s not found" % str(datalogger).split("%")[0])
datalogger += "%%%s" % (float(dte.gain) * float(gaind))
print(" Converting gain multiplier to real gain using instrument DB on %s" % code, file=sys.stderr)
except Exception as e:
raise Exception ("Missing Gain on %s (%s)" % (line,str(e)))
try:
channel = items.pop(0)
except Exception as e:
raise Exception ("Missing Channel on %s" % line)
try:
latitude = items.pop(0)
except Exception as e:
raise Exception ("Missing Latitude on %s" % line)
try:
longitude = items.pop(0)
except Exception as e:
raise Exception ("Missing Longitude on %s" % line)
try:
elevation = items.pop(0)
except Exception as e:
raise Exception ("Missing Elevation on %s" % line)
try:
depth = items.pop(0)
except Exception as e:
raise Exception ("Missing Depth on %s" % line)
#Orientation
try:
float(depth)
orientation = "ZNE"
except:
orientation = "Z"
(depth,a1,a2) = depth.split("/")
a1n = float(a1)
if a1n == 0.0:
orientation+="1"
else:
orientation+="1(0.0,%s)"%a1
a2n = float(a2)
if a2n == 90.0:
orientation+="2"
else:
orientation+="2(0.0,%s)"%a2
# Start
try:
start = items.pop(0)
except Exception:
raise Exception ("Missing Start on %s" % line)
try:
start = parseDate(start)
except Exception as e:
raise Exception("Invalide Start date: %s (%s) on %s" % (start, e, line))
#End
try:
end = items.pop(0)
except:
end = ""
try:
end = parseDate(end)
except Exception as e:
raise Exception("Invalide End date: %s (%s) on %s" % (end, e, line))
[place, country] = self.sat.parseStationLine(line.split())
description = "%s/%s" % (place, country)
## Prepare necessary output
if not atFront:
self.sma.dump(fdo, code)
self.sat.dump(fdo, code)
for (start, end) in self.sma.getMappings(code, start, end):
fdo.write(lnfmt % (code, quote(description), datalogger, sensor, channel, orientation, latitude, longitude, elevation, depth, formatDate(start), formatDate(end)) + "\n")
return code
def convert(self, fdo, keepcomments = False, atFront = True):
if self.filename is None:
raise Exception("You should pre-load a tab file before before converting.")
## Obtain additional attribute classes if needed
if not self.nat:
self.nat = NetworkAttributes(self.networkCode, None)
if not self.sat:
self.sat = StationAttributes(self.networkCode, self.stationList, None)
if not self.sma:
self.sma = StationMappings(self.networkCode, self.stationList, None)
# Parse in again the station lines and network header by the additional classes
print("Pre-Parsing Station/Network lines ... ", file=sys.stderr)
fd = open(self.filename)
for line in fd:
line = line.strip()
if not line or line[0] == "#":
continue
items = line.split()
if len(items) <= 5:
self.nat.parseNetworkLine(items)
elif len(items) <= 12:
self.sma.parseStationLine(items)
self.sat.parseStationLine(items)
fd.close()
fd = open(self.filename)
oldcode="" # Station code of the last printed line
last="" # Type of the last printed line
print("Converting ... ", file=sys.stderr)
for line in fd:
line = line.strip()
if not line or line[0] == "#":
if last == "l" or last == "a" or last == "h": fdo.write("\n")
if keepcomments: fdo.write(line + "\n")
last = "c"
continue
items = line.split()
if len(items) <= 5:
self.__convertHeader__(line, fdo)
last = "h"
if (atFront):
fdo.write("\n")
self.sma.dump(fdo, None)
self.sat.dump(fdo, None)
last = "a"
fdo.write("\n")
elif len(items) <= 12:
if (last == "l" and items[0].strip() != oldcode) or last == "h": fdo.write("\n")
oldcode = self.__convertLine__(line, fdo, atFront)
last = "l"
pass
else:
print("input at %s" % line, file=sys.stderr)
fd.close()
def main():
# Creating the parser
parser = OptionParser(usage="Old tab to New tab converter", version="1.0", add_help_option=True)
parser.add_option("", "--instdb", type="string",
help="Indicates the instrument databases file to use", dest="inst", default=None)
parser.add_option("", "--smap", type="string",
help="Indicates the station attribute file to use", dest="smap", default=None)
parser.add_option("", "--sat", type="string",
help="Indicates the station attribute file to use", dest="sat", default=None)
parser.add_option("", "--nat", type="string",
help="Indicates the station attribute file to use", dest="nat", default=None)
parser.add_option("-t", "--tab", type="string",
help="Indicates the tab file to convert", dest="tabFile", default=None)
parser.add_option("-f", "--filterf", type="string",
help="Indicates a folder containing the filters coefficients files", dest="ffolder", default=None)
parser.add_option("-n", "--net", type="string",
help="Indicates a two leter station code", dest="netCode", default=None)
parser.add_option("-g", "--globalsa", action="store_true",
help="Indicate that we should put a condensed version of the station attributes just below the network definition", dest="globalSa", default=False)
parser.add_option("-a", "--autotime", action="store_true",
help="Guess the start and end times for a network from the channel times", dest="autoTime", default=False)
parser.add_option("-c", "--clean", action="store_true",
help="Remove the comments and blank lines", dest="cleanFile", default=False)
# Parsing & Error check
(options, args) = parser.parse_args()
error = False
if len(args) != 1:
print("need an Output Filename or '-' for stdout", file=sys.stderr)
error = True
if not options.tabFile:
print("tab file name not supplied", file=sys.stderr)
error = True
if options.inst and not options.ffolder:
print("Filter folder not supplied.", file=sys.stderr)
error = True
if options.tabFile and not os.path.isfile(options.tabFile):
print("supplied tab file (%s) is not a file" % options.tabFile, file=sys.stderr)
error = True
if not options.netCode:
print("network code not supplied", file=sys.stderr)
error = True
#if options.autoTime and (options.netStart or options.netEnd):
# print >> sys.stderr, "options Auto Time and Network Start/End times are exclusive"
# return
if error:
print("use -h for getting a help on usage", file=sys.stderr)
return
if args[0] != "-":
fdo = open(args[0], "w")
else:
fdo = sys.stdout
# Execution
try:
cnv = TabConverter(options.netCode.upper())
cnv.preload(options.tabFile, options.autoTime)
if options.inst or options.smap or options.nat or options.sat:
print("Loading optional files: ", file=sys.stderr)
if options.inst and os.path.isfile(options.inst):
cnv.loadInstrumentsFile(options.inst, options.ffolder)
if options.smap and os.path.isfile(options.smap):
cnv.loadStationMapping(options.smap)
if options.nat and os.path.isfile(options.nat):
cnv.loadNetworkAttribute(options.nat)
if options.sat and os.path.isfile(options.sat):
cnv.loadStationAttribute(options.sat)
print("Done.", file=sys.stderr)
cnv.convert(fdo, not options.cleanFile, options.globalSa)
except Exception as e:
print("", file=sys.stderr)
print("Error on processing: %s" % e, file=sys.stderr)
fdo.close()
if __name__ == "__main__":
main()