527 lines
18 KiB
Plaintext
Executable File
527 lines
18 KiB
Plaintext
Executable File
#!/usr/bin/env seiscomp-python
|
|
|
|
from __future__ import print_function
|
|
import os
|
|
import sys
|
|
from datetime import datetime
|
|
from nettab.convertUtils import StationAttributes, NetworkAttributes, StationMappings, parseDate, formatDate, quote, hummanStr
|
|
from nettab.tab import Tab
|
|
from optparse import OptionParser
|
|
from nettab.nodesi import Instruments
|
|
|
|
class TabConverter:
|
|
def __init__(self, networkCode):
|
|
self.__fmt__ = None
|
|
self.takeSugestions = None
|
|
|
|
self.filename = None
|
|
|
|
self.networkCode = networkCode
|
|
self.stationList = None
|
|
|
|
self.nat = None
|
|
self.sat = None
|
|
self.sma = None
|
|
self.inst = None
|
|
self.defaultEpoch = parseDate("1980/001")
|
|
|
|
self.start=0
|
|
self.code=0
|
|
self.description=0
|
|
self.datalogger=0
|
|
self.sensor=0
|
|
self.channel=0
|
|
self.gaind = 0
|
|
self.longitude=0
|
|
self.latitude=0
|
|
self.elevation=0
|
|
self.end=0
|
|
self.depth=0
|
|
self.orientation=0
|
|
|
|
## default dates
|
|
self.startDate = parseDate("1980/001")
|
|
self.endDate = parseDate(None)
|
|
|
|
def loadStationMapping(self, filename):
|
|
if self.networkCode is None: raise Exception("Cannot load Station mapping without network code")
|
|
if self.stationList is None: raise Exception("Cannot load Station mapping without station list")
|
|
|
|
try:
|
|
sm = StationMappings(self.networkCode, self.stationList, filename)
|
|
self.sma = sm
|
|
except Exception as e:
|
|
raise e
|
|
|
|
def loadStationAttribute(self, filename):
|
|
if self.networkCode is None: raise Exception("Cannot load Station att without network code")
|
|
if self.stationList is None: raise Exception("Cannot load Station att without station list")
|
|
|
|
try:
|
|
sa = StationAttributes(self.networkCode, self.stationList, filename)
|
|
self.sat = sa
|
|
except Exception as e:
|
|
raise e
|
|
|
|
def loadNetworkAttribute(self, filename):
|
|
if self.networkCode is None: raise Exception("Cannot load Network att without network code")
|
|
if self.stationList is None: raise Exception("Cannot load Network att without station list")
|
|
try:
|
|
na = NetworkAttributes(self.networkCode, filename)
|
|
self.nat = na
|
|
except Exception as e:
|
|
raise e
|
|
|
|
def loadInstrumentsFile(self, filename, filterFolder):
|
|
tab = Tab(filterFolder=filterFolder)
|
|
tab.digest(filename)
|
|
if tab.i:
|
|
self.inst = tab.i
|
|
|
|
def __fmtline__(self):
|
|
if not self.__fmt__:
|
|
fmt = "Sl: "
|
|
fmt += "%%-%ds" % self.code
|
|
fmt += " %%-%ds" % self.description
|
|
fmt += " %%-%ds" % self.datalogger
|
|
fmt += " %%-%ds" % self.sensor
|
|
fmt += " %%-%ds" % self.channel
|
|
fmt += " %%-%ds" % self.orientation
|
|
fmt += " %%-%ds" % self.latitude
|
|
fmt += " %%-%ds" % self.longitude
|
|
fmt += " %%-%ds" % self.elevation
|
|
fmt += " %%-%ds" % self.depth
|
|
fmt += " %%-%ds" % self.start
|
|
fmt += " %%-%ds" % self.end
|
|
self.__fmt__ = fmt
|
|
|
|
return self.__fmt__
|
|
|
|
def __analyseLine__(self, items):
|
|
inputLine = " ".join(items)
|
|
if len(items) < 4:
|
|
raise Exception("Invalid items count on line %s" % inputLine)
|
|
|
|
if len(items) <= 5:
|
|
netCode = items[2]
|
|
if netCode != self.networkCode:
|
|
raise Exception("Tab file (%s) doesn't match class (%s) -- %s" % (netCode,self.networkCode,inputLine))
|
|
return [None, None, None]
|
|
else:
|
|
if len(items) < 6:
|
|
raise Exception("Invalid Station line %s" % inputLine)
|
|
|
|
stationCode = items.pop(0)
|
|
code = len(stationCode)
|
|
self.code=max(self.code,code)
|
|
|
|
description = len(quote(hummanStr(items.pop(0))))
|
|
self.description=max(self.description, description)
|
|
|
|
datalogger = len(items.pop(0))
|
|
self.datalogger=max(self.datalogger, datalogger)
|
|
|
|
sensor = len(items.pop(0))
|
|
self.sensor=max(self.sensor, sensor)
|
|
|
|
# Gain
|
|
gaind = items.pop(0)
|
|
if float(gaind) != 1.0:
|
|
self.datalogger = max (self.datalogger, datalogger + len(gaind))
|
|
|
|
channel = len(items.pop(0))
|
|
self.channel=max(self.channel, channel)
|
|
|
|
latitude = len(items.pop(0))
|
|
self.latitude=max(self.latitude, latitude)
|
|
|
|
longitude = len(items.pop(0))
|
|
self.longitude=max(self.longitude, longitude)
|
|
|
|
elevation = len(items.pop(0))
|
|
self.elevation=max(self.elevation, elevation)
|
|
|
|
#Orientation
|
|
depth = items.pop(0)
|
|
try:
|
|
float(depth)
|
|
orientation="ZNE"
|
|
except:
|
|
orientation = "Z"
|
|
(depth,a1,a2) = depth.split("/")
|
|
|
|
a1n = float(a1)
|
|
a2n = float(a2)
|
|
|
|
orientation+="1"
|
|
if a1n != 0.0: orientation += "(0.0,%s)"%a1
|
|
|
|
orientation+="2"
|
|
if a2n != 90.0: orientation+="(0.0,%s)"%a1
|
|
|
|
orientation = len(orientation)
|
|
self.orientation=max(self.orientation, orientation)
|
|
|
|
depth = len(depth)
|
|
self.depth=max(self.depth, depth)
|
|
|
|
# Start
|
|
try:
|
|
start = parseDate(items.pop(0))
|
|
self.start = max (self.start, len(formatDate(start)))
|
|
except:
|
|
raise Exception ("Invalid Station line start date %s" % inputLine)
|
|
|
|
# End
|
|
try:
|
|
end = parseDate(items.pop(0))
|
|
except:
|
|
end=parseDate("")
|
|
pass
|
|
self.end = max (self.end, len(formatDate(end)))
|
|
|
|
return [stationCode, start, end]
|
|
|
|
def preload(self, filename, takeSugestions):
|
|
self.takeSugestions = takeSugestions
|
|
sugestedStart = datetime.now()
|
|
sugestedEnd = self.defaultEpoch
|
|
stationList = []
|
|
|
|
error = []
|
|
|
|
# Some initialization
|
|
if self.filename is not None:
|
|
raise Exception("Cannot pre-load two different files (current one is %s)" % self.filename)
|
|
|
|
print("Analysing ... ", file=sys.stderr)
|
|
fd = open(filename)
|
|
for line in fd:
|
|
line = line.strip()
|
|
if not line or line[0] == "#": continue
|
|
|
|
try:
|
|
(stationCode, start, end) = self.__analyseLine__(line.split())
|
|
except Exception as e:
|
|
error.append(str(e))
|
|
continue
|
|
|
|
if not stationCode: continue
|
|
if stationCode not in stationList:
|
|
stationList.append(stationCode)
|
|
|
|
sugestedStart = min(sugestedStart, start)
|
|
if end and sugestedEnd:
|
|
sugestedEnd = max(sugestedEnd, end)
|
|
else:
|
|
sugestedEnd = None
|
|
fd.close()
|
|
|
|
if len(error):
|
|
raise Exception("\n".join(error))
|
|
|
|
print(" Loaded %d different stations" % len(stationList), file=sys.stderr)
|
|
if takeSugestions:
|
|
print(" Taking suggestion start date of %s " % formatDate(self.startDate), file=sys.stderr)
|
|
self.startDate = sugestedStart
|
|
print(" Taking suggestion end date of %s " % formatDate(self.endDate), file=sys.stderr)
|
|
self.endDate = sugestedEnd
|
|
|
|
self.filename = filename
|
|
self.stationList = stationList
|
|
print("Done.", file=sys.stderr)
|
|
|
|
def __convertHeader__(self, line, fdo):
|
|
|
|
# Split line
|
|
items = line.split()
|
|
|
|
if not self.takeSugestions:
|
|
if self.nat.hasStart:
|
|
print(" Using start from attribute.", file=sys.stderr)
|
|
self.startDate = self.nat.startDate
|
|
if self.nat.hasEnd:
|
|
print(" Using end from attribute.", file=sys.stderr)
|
|
self.endDate = self.nat.endDate
|
|
|
|
nCode = items[2].strip()
|
|
if nCode != self.networkCode:
|
|
raise Exception("Wrong network code found: %s != %s" % (self.networkCode, nCode))
|
|
|
|
fdo.write("Nw: %s %s %s" % (nCode, formatDate(self.startDate), formatDate(self.endDate)) + "\n")
|
|
|
|
self.nat.dump(fdo)
|
|
|
|
def __convertLine__(self, line, fdo, atFront):
|
|
lnfmt = self.__fmtline__()
|
|
|
|
# Split line
|
|
items = line.split()
|
|
|
|
try:
|
|
code = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Code on %s" % line)
|
|
|
|
if code not in self.stationList:
|
|
raise Exception("Unknow station code $s" % code)
|
|
|
|
try:
|
|
hummanStr(items.pop(0))
|
|
except Exception as e:
|
|
raise Exception ("Missing Gain on %s" % line)
|
|
|
|
try:
|
|
datalogger = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Datalogger on %s" % line)
|
|
|
|
try:
|
|
sensor = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Sensor on %s" % line)
|
|
|
|
try:
|
|
gaind = items.pop(0)
|
|
if float(gaind) != 1.0:
|
|
if not self.inst:
|
|
raise Exception("Instrument database needed to convert gain")
|
|
try:
|
|
dte = self.inst.dls[str(datalogger).split("%")[0]]
|
|
except Exception as e:
|
|
print(e, file=sys.stderr)
|
|
raise Exception("Datalogger %s not found" % str(datalogger).split("%")[0])
|
|
datalogger += "%%%s" % (float(dte.gain) * float(gaind))
|
|
print(" Converting gain multiplier to real gain using instrument DB on %s" % code, file=sys.stderr)
|
|
except Exception as e:
|
|
raise Exception ("Missing Gain on %s (%s)" % (line,str(e)))
|
|
|
|
try:
|
|
channel = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Channel on %s" % line)
|
|
|
|
try:
|
|
latitude = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Latitude on %s" % line)
|
|
|
|
try:
|
|
longitude = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Longitude on %s" % line)
|
|
try:
|
|
elevation = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Elevation on %s" % line)
|
|
|
|
try:
|
|
depth = items.pop(0)
|
|
except Exception as e:
|
|
raise Exception ("Missing Depth on %s" % line)
|
|
|
|
#Orientation
|
|
try:
|
|
float(depth)
|
|
orientation = "ZNE"
|
|
except:
|
|
orientation = "Z"
|
|
(depth,a1,a2) = depth.split("/")
|
|
|
|
a1n = float(a1)
|
|
if a1n == 0.0:
|
|
orientation+="1"
|
|
else:
|
|
orientation+="1(0.0,%s)"%a1
|
|
|
|
a2n = float(a2)
|
|
if a2n == 90.0:
|
|
orientation+="2"
|
|
else:
|
|
orientation+="2(0.0,%s)"%a2
|
|
|
|
# Start
|
|
try:
|
|
start = items.pop(0)
|
|
except Exception:
|
|
raise Exception ("Missing Start on %s" % line)
|
|
|
|
try:
|
|
start = parseDate(start)
|
|
except Exception as e:
|
|
raise Exception("Invalide Start date: %s (%s) on %s" % (start, e, line))
|
|
|
|
#End
|
|
try:
|
|
end = items.pop(0)
|
|
except:
|
|
end = ""
|
|
|
|
try:
|
|
end = parseDate(end)
|
|
except Exception as e:
|
|
raise Exception("Invalide End date: %s (%s) on %s" % (end, e, line))
|
|
|
|
[place, country] = self.sat.parseStationLine(line.split())
|
|
description = "%s/%s" % (place, country)
|
|
|
|
## Prepare necessary output
|
|
if not atFront:
|
|
self.sma.dump(fdo, code)
|
|
self.sat.dump(fdo, code)
|
|
|
|
for (start, end) in self.sma.getMappings(code, start, end):
|
|
fdo.write(lnfmt % (code, quote(description), datalogger, sensor, channel, orientation, latitude, longitude, elevation, depth, formatDate(start), formatDate(end)) + "\n")
|
|
|
|
return code
|
|
|
|
def convert(self, fdo, keepcomments = False, atFront = True):
|
|
if self.filename is None:
|
|
raise Exception("You should pre-load a tab file before before converting.")
|
|
|
|
## Obtain additional attribute classes if needed
|
|
if not self.nat:
|
|
self.nat = NetworkAttributes(self.networkCode, None)
|
|
if not self.sat:
|
|
self.sat = StationAttributes(self.networkCode, self.stationList, None)
|
|
if not self.sma:
|
|
self.sma = StationMappings(self.networkCode, self.stationList, None)
|
|
|
|
# Parse in again the station lines and network header by the additional classes
|
|
print("Pre-Parsing Station/Network lines ... ", file=sys.stderr)
|
|
fd = open(self.filename)
|
|
for line in fd:
|
|
line = line.strip()
|
|
if not line or line[0] == "#":
|
|
continue
|
|
items = line.split()
|
|
if len(items) <= 5:
|
|
self.nat.parseNetworkLine(items)
|
|
elif len(items) <= 12:
|
|
self.sma.parseStationLine(items)
|
|
self.sat.parseStationLine(items)
|
|
fd.close()
|
|
|
|
fd = open(self.filename)
|
|
oldcode="" # Station code of the last printed line
|
|
last="" # Type of the last printed line
|
|
print("Converting ... ", file=sys.stderr)
|
|
for line in fd:
|
|
line = line.strip()
|
|
if not line or line[0] == "#":
|
|
if last == "l" or last == "a" or last == "h": fdo.write("\n")
|
|
if keepcomments: fdo.write(line + "\n")
|
|
last = "c"
|
|
continue
|
|
items = line.split()
|
|
if len(items) <= 5:
|
|
self.__convertHeader__(line, fdo)
|
|
last = "h"
|
|
if (atFront):
|
|
fdo.write("\n")
|
|
self.sma.dump(fdo, None)
|
|
self.sat.dump(fdo, None)
|
|
last = "a"
|
|
fdo.write("\n")
|
|
elif len(items) <= 12:
|
|
if (last == "l" and items[0].strip() != oldcode) or last == "h": fdo.write("\n")
|
|
oldcode = self.__convertLine__(line, fdo, atFront)
|
|
last = "l"
|
|
pass
|
|
else:
|
|
print("input at %s" % line, file=sys.stderr)
|
|
fd.close()
|
|
|
|
def main():
|
|
# Creating the parser
|
|
parser = OptionParser(usage="Old tab to New tab converter", version="1.0", add_help_option=True)
|
|
|
|
parser.add_option("", "--instdb", type="string",
|
|
help="Indicates the instrument databases file to use", dest="inst", default=None)
|
|
parser.add_option("", "--smap", type="string",
|
|
help="Indicates the station attribute file to use", dest="smap", default=None)
|
|
parser.add_option("", "--sat", type="string",
|
|
help="Indicates the station attribute file to use", dest="sat", default=None)
|
|
parser.add_option("", "--nat", type="string",
|
|
help="Indicates the station attribute file to use", dest="nat", default=None)
|
|
parser.add_option("-t", "--tab", type="string",
|
|
help="Indicates the tab file to convert", dest="tabFile", default=None)
|
|
parser.add_option("-f", "--filterf", type="string",
|
|
help="Indicates a folder containing the filters coefficients files", dest="ffolder", default=None)
|
|
parser.add_option("-n", "--net", type="string",
|
|
help="Indicates a two leter station code", dest="netCode", default=None)
|
|
parser.add_option("-g", "--globalsa", action="store_true",
|
|
help="Indicate that we should put a condensed version of the station attributes just below the network definition", dest="globalSa", default=False)
|
|
parser.add_option("-a", "--autotime", action="store_true",
|
|
help="Guess the start and end times for a network from the channel times", dest="autoTime", default=False)
|
|
parser.add_option("-c", "--clean", action="store_true",
|
|
help="Remove the comments and blank lines", dest="cleanFile", default=False)
|
|
|
|
# Parsing & Error check
|
|
(options, args) = parser.parse_args()
|
|
error = False
|
|
|
|
if len(args) != 1:
|
|
print("need an Output Filename or '-' for stdout", file=sys.stderr)
|
|
error = True
|
|
|
|
if not options.tabFile:
|
|
print("tab file name not supplied", file=sys.stderr)
|
|
error = True
|
|
|
|
if options.inst and not options.ffolder:
|
|
print("Filter folder not supplied.", file=sys.stderr)
|
|
error = True
|
|
|
|
if options.tabFile and not os.path.isfile(options.tabFile):
|
|
print("supplied tab file (%s) is not a file" % options.tabFile, file=sys.stderr)
|
|
error = True
|
|
|
|
if not options.netCode:
|
|
print("network code not supplied", file=sys.stderr)
|
|
error = True
|
|
|
|
#if options.autoTime and (options.netStart or options.netEnd):
|
|
# print >> sys.stderr, "options Auto Time and Network Start/End times are exclusive"
|
|
# return
|
|
|
|
if error:
|
|
print("use -h for getting a help on usage", file=sys.stderr)
|
|
return
|
|
|
|
if args[0] != "-":
|
|
fdo = open(args[0], "w")
|
|
else:
|
|
fdo = sys.stdout
|
|
|
|
# Execution
|
|
try:
|
|
cnv = TabConverter(options.netCode.upper())
|
|
cnv.preload(options.tabFile, options.autoTime)
|
|
|
|
if options.inst or options.smap or options.nat or options.sat:
|
|
print("Loading optional files: ", file=sys.stderr)
|
|
|
|
if options.inst and os.path.isfile(options.inst):
|
|
cnv.loadInstrumentsFile(options.inst, options.ffolder)
|
|
|
|
if options.smap and os.path.isfile(options.smap):
|
|
cnv.loadStationMapping(options.smap)
|
|
|
|
if options.nat and os.path.isfile(options.nat):
|
|
cnv.loadNetworkAttribute(options.nat)
|
|
|
|
if options.sat and os.path.isfile(options.sat):
|
|
cnv.loadStationAttribute(options.sat)
|
|
print("Done.", file=sys.stderr)
|
|
|
|
cnv.convert(fdo, not options.cleanFile, options.globalSa)
|
|
except Exception as e:
|
|
print("", file=sys.stderr)
|
|
print("Error on processing: %s" % e, file=sys.stderr)
|
|
|
|
fdo.close()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|