#!/usr/bin/env -S python3 -B
#
#   ft8report - lightweight client for PSKreporter
#
#   The program can read log messages from an ALL.TXT file, such as
#   from ft8cat's -a option, or from a UDP log source, such as from
#   ft8cat's -A option.  The code can currently only consume one or
#   the other, not both.
#
#   The -a option is intended for sites that want to both upload
#   and locally record spots.  The -A option is for sites that only
#   want to upload spots, but not record them.
#
#   Messages are parsed as they arrive, but outgoing updates to the
#   PSKreporter site are done approximately one minute after
#   starting, and at five-minute intervals thereafter.
#
#   This is a from-scratch implementation, based on information
#   from the developer's reference material at PSKreporter.info.
#
#   See: https://pskreporter.info/pskdev.html
#
#   Copyright (C) 2024 by Matt Roberts.
#   License: GNU GPL3 (www.gnu.org)
#
#


# system modules
import socket
import getopt
import select
import random
import signal
import syslog
import atexit
import time
import sys
import os
import io

# local modules
from conflicts import *
from fdutils import *
from fragment import *
from parsing import *
from spots import *
from version import *

# the ALL.TXT follower
import tailall


#
#  settings
#
path = None        # the path to ALL.TXT (option -a)
mycall = None      # the receiving callsign (option -c)
mygrid = None      # the receiving grid square (option -g)
verbose = 0        # verbosity (option -v)
background = False # when true, run as a background service
minhits = 2        # the number of times a station needs to be seen before reported
dryrun = False     # when true, skip the UDP uploads themselves

# host and port
host = 'report.pskreporter.info' # new reporting host
port = 4739                      # production port (option -t)

# reporting interval
interval = 300 # seconds

# the client ID - randomly generated
client_id = random.randint(0, 0x7FFFFFFF)

# number of calls (max) to include in each datagram
cut_size = 40


#
#  global state
#
spots = { }     # cache of spots; { call -> { ... data ... } )
update = 0      # the most recent update
sequence = 0    # the sequence number
udp = None      # the socket


#
#  usage() - show the command line options
#
def usage():
	global minhits

	sys.stdout.write("\n")
	sys.stdout.write("Usage: %s [options]\n" % sys.argv[0])
	sys.stdout.write("\n")
	sys.stdout.write("Follow an ALL.TXT file generated by the 'ft8cat' utility or similar,\n")
	sys.stdout.write("and upload spots periodically to PSKreporter's website.\n")
	sys.stdout.write("\n")
	sys.stdout.write("Options include:\n")
	sys.stdout.write("\t-a <path> - read data from file at 'path'\n")
	sys.stdout.write("\t-A <port> - read data from UDP port, instead of file; the argument\n")
	sys.stdout.write("\t            can be a port number, or ip:port, or host:port\n")
	sys.stdout.write("\t-c <call> - specify the receiving callsign\n")
	sys.stdout.write("\t-g <grid> - specify the receiving grid square\n")
	sys.stdout.write("\t-n <hits> - minimum number of spots from a callsign before it is\n")
	sys.stdout.write("\t            reported to the website (default: 2)\n")
	sys.stdout.write("\t-i <id>   - set the client ID; (default: auto-generated)\n")
	sys.stdout.write("\t-d        - increase debugging output\n")
	sys.stdout.write("\t-t        - use the test server at pskreporter.info\n")
	sys.stdout.write("\t-r        - dry-run; do everything else but the UDP uplaods\n")
	sys.stdout.write("\t-h        - show this information\n")
	sys.stdout.write("\t-v        - show the version number\n")
	sys.stdout.write("\t-z        - run in background\n")
	sys.stdout.write("\n")
	sys.stdout.write("Either the -a or -A option is required, but not both.\n")
	sys.stdout.write("\n")
	sys.stdout.write("Use of -i is optional, but the client ID value is persisted to the\n")
	sys.stdout.write("file ~/.ft8reportrc, and will be used until changed by use of -i.\n")
	sys.stdout.write("\n")
	sys.stdout.write("The -c, and -g options are required.  The values are also persisted\n")
	sys.stdout.write("to ~/.ft8reportrc, and can be omitted after being provided once.\n")
	sys.stdout.write("\n")
	sys.stdout.write("The -c and -g option values are sent to the pskreporter.info website\n")
	sys.stdout.write("within UDP frame, and will be publicly visible on that website, both\n")
	sys.stdout.write("in the map view, and in the statistics.  Only supply values that you\n")
	sys.stdout.write("want other people to see.\n")
	sys.stdout.write("\n")
	sys.exit(1)


#
#  MHz() - convert MHz to Hz
#
def MHz(f):
	return int(f / 1000000.0)


#
#  network_bytes(n, m)
#
#  return network byte order of integer 'n's least-significant 'm' bytes
#
#  This is a conversion utility for building UDP datagrams.
#
def network_bytes(n, m):
	if m == 4:
		n = socket.htonl(n)
	elif m == 2:
		n = socket.htons(n)
	else:
		raise Exception("network_bytes: invalid 'm' value")
	result = [ ]
	for i in range(m):
		result.append(int(n & 0xFF))
		n >>= 8
	return result


#
#  string_bytes(s)
#
#  return length-led sequence representing a string
#
#  This is a conversion utility for building UDP datagrams.
#
def string_bytes(s):
	result = [ ]
	result.append(len(s))
	for c in s:
		result.append(int(ord(c) & 0xFF))
	return result


#
#  pad_block(b, n) - pad a block of values 'b' up to a size divisible by 'n'
#
#  This is a conversion utility for building UDP datagrams.
#
def pad_block(b, n = 4):
	r = len(b) % n
	pad = [0] * (n - r)
	return b + pad


#
#  save_config() - write rc file at exit
#
def save_config():
	global client_id, mycall, mygrid

	# save configuration
	fn = os.path.join(os.path.expanduser('~'), '.ft8reportrc')
	with io.open(fn, 'w') as f:
		f.write("ID %d\n" % client_id)
		f.write("CALL %s\n" % mycall)
		f.write("GRID %s\n" % mygrid)

		# DEBUG:
		if verbose > 1:
			sys.stderr.write("DEBUG: Wrote saved state: %s\n" % fn)


#
#  build_datagram(cut)
#
#  This is the main datagram construction function; the 'cut' is a
#  list of spots, including call and supporting data, to be encoded
#  into a single datagram; 'now' is the current clock as an integer.
#
#  This function does not cut a set of spots into smaller groups;
#  that is provided elsewhere.
#
#  The datagram is built up as an array of integers, all less than 0xFF,
#  which is converted into a 'bytes' object at the very end.
#
def build_datagram(cut, now):
	global verbose, background, sequence, client_id, mycall, mygrid

	# start with the 'header' portion
	datagram = [ ]
	datagram += [ 0x00, 0x0A, 0x00, 0x00 ]
	datagram += network_bytes(int(now), 4) # the timestamp
	datagram += network_bytes(sequence, 4) # the sequence #
	datagram += network_bytes(client_id, 4) # the client ID

	# next, the "receiver information descriptor"
	datagram += [
		# this version is for (call, grid, software)
		0x00, 0x03, 0x00, 0x24, 0x99, 0x92, 0x00, 0x03, 0x00, 0x00,
		0x80, 0x02, 0xFF, 0xFF, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x04, 0xFF, 0xFF, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x08, 0xFF, 0xFF, 0x00, 0x00, 0x76, 0x8F,
		0x00, 0x00 ]
	
	# next, the "sender information descriptor"
	datagram += [
		# this version is for ( call, freq, SNR (1 byte), IMD (=0), mode, src (=1), grid, time )
		0x00, 0x02, 0x00, 0x44, 0x99, 0x93, 0x00, 0x08,
		0x80, 0x01, 0xFF, 0xFF, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x05, 0x00, 0x04, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x06, 0x00, 0x01, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x07, 0x00, 0x01, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x0A, 0xFF, 0xFF, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x0B, 0x00, 0x01, 0x00, 0x00, 0x76, 0x8F,
		0x80, 0x03, 0xFF, 0xFF, 0x00, 0x00, 0x76, 0x8F,
		0x00, 0x96, 0x00, 0x04
	]

	# next, build the "receiver information record" block
	software = "KK5JY ft8report " + GetModemVersion()
	receiver = [ 0x99, 0x92, 0x00, 0x00 ]
	receiver += string_bytes(mycall)   # my callsign
	receiver += string_bytes(mygrid)   # my grid square
	receiver += string_bytes(software) # the software version string
	receiver = pad_block(receiver)     # pad up the block
	length = network_bytes(len(receiver), 2)
	receiver[2:4] = length             # update the length
	datagram += receiver               # append to the datagram

	# next, build the "sender information record(s)" block
	senders = [ 0x99, 0x93, 0x00, 0x00 ]
	for spot in cut:
		senders += string_bytes(spot['call'])       # their call
		senders += network_bytes(spot['freq'], 4)   # RF
		senders.append(spot['snr'] & 0xFF)          # SNR
		senders.append(0)                           # IMD
		senders += string_bytes(spot['mode'])       # mode
		senders.append(1)                           # src (auto)
		senders += string_bytes(spot['grid'])       # grid
		senders += network_bytes(spot['when'], 4)   # when (of the spot)
	senders = pad_block(senders)                    # pad up the block
	length = network_bytes(len(senders), 2)
	senders[2:4] = length                           # update the length
	datagram += senders                             # append to the datagram

	# finally, update the overall length field in the header
	length = network_bytes(len(datagram), 2)
	datagram[2:4] = length

	# now convert the whole thing to a 'bytes' object
	datagram = bytes(datagram)

	# DEBUG: short version
	if verbose:
		sys.stderr.write("DEBUG: Datagram has %d bytes:" % len(datagram))
		if verbose < 2:
			sys.stderr.write("\n")

	# DEBUG: long version
	if verbose >= 2:
		ct = 0
		for b in datagram:
			if ct % 16 == 0:
				sys.stderr.write('\n\t')
			sys.stderr.write("%02X " % b)
			ct += 1
		sys.stderr.write('\n')

	# all done
	return datagram


#
#  process_spot(d) - process new spot from the callback
#
def process_spot(d):
	global verbose, mycall, spots

	# if no spot, just return to the callback
	if not d:
		return

	# pull out the fields we care about, and convert a few of them
	tx = d['tx']
	fr = d['from']
	to = d['to']
	gr = d['grid']
	snr = int(d['snr'])
	mode = d['mode']
	audio = d['audio']
	freq = d['freq']
	when = int(d['when'])

	# skip TX frames
	if tx:
		return

	# skip my own call as source, even in reception reports
	if fr == mycall:
		return

	# skip empty calls
	if not fr:
		return
	
	# skip incomplete/hashed calls
	if '.' in fr:
		return

	# compute the real RF frequency from the dial and AF
	rf = int(int(audio) + (float(freq) * 1000000.0))

	# if this is a new call...
	if fr not in spots:
		# ...add a basic record with some fields missing
		spots[fr] = { 'mode' : mode, 'freq' : rf, 'snr' : snr, 'grid' : '', 'count' : 0 }

	# see if there has been a band/mode change
	reset = False
	if spots[fr]['mode'] != mode:
		reset = True
	if MHz(spots[fr]['freq']) != MHz(rf):
		reset = True

	# do selective updates
	spots[fr]['mode'] = mode
	spots[fr]['freq'] = rf
	spots[fr]['when'] = when
	spots[fr]['count'] = 1 + (0 if reset else spots[fr]['count'])
	if reset or spots[fr]['snr'] is None or snr > spots[fr]['snr']:
		spots[fr]['snr'] = snr
	if gr:
		spots[fr]['grid'] = gr

	# DEBUG:
	if verbose >= 2:
		rgrid = spots[fr]['grid']
		if not rgrid:
			rgrid = '...'
		rdest = to
		if not rdest:
			rdest = '...'
		sys.stderr.write(
			"DEBUG: %s %d %s -> %s; best SNR = %d; grid = %s; time = %d\n" % (
				mode, rf, fr, rdest, spots[fr]['snr'], rgrid, int(time.time() - when)))


#
#  upload_cycle(...) - process calls for upload
#
def upload_cycle(now):
	global verbose, mycall, mygrid, path, host, port, client_id, cut_size, dryrun, minhits
	global spots, sequence, update, udp

	# get the list of calls that can be reported
	reportable = [ ]
	to_remove = [ ]
	for call in spots.keys():
		spot = spots[call]
		# select stations seen since last UDP update, and with enough spots
		if spot['when'] > update and spot['count'] >= minhits:
			reportable.append({
				'call': call,
				'mode': spot['mode'],
				'freq': spot['freq'],
				'when': spot['when'],
				'grid': spot['grid'],
				'snr': spot['snr']
			})

		# also clear out old spots, to keep the dictionary a reasonable size
		if now - spot['when'] >= 86400: # 1 day
			to_remove.append(call)

	# remove the old ones
	for call in to_remove:
		del spots[call]

	# SYSLOG and DEBUG
	if verbose or background:
		# calculate elapsed time since last update
		elapsed = now - update

		# build the message
		msg = "Send %d of %d calls from cycle of %d seconds" % (len(reportable), len(spots), elapsed)

		# DEBUG:
		if verbose:
			sys.stderr.write("DEBUG: %s\n" % msg)

		# SYSLOG:
		if background:
			syslog.syslog(msg)

	# set the last-update time to 'now'
	update = now

	#
	#  UDP: build and upload the report UDP frame(s); if the number of
	#       spot records exceeds 'cut_size', the list will be broken
	#       into a set of datagrams with no more than 'cut_size'
	#       records per datagram, to prevent MTU/fragmentation issues.
	#
	#  NOTE: if 'reportable' is empty, 'frags' will also be empty
	#
	frags = fragment(reportable, cut_size)
	for frag in frags:
		# build the datagram for this cut
		datagram = build_datagram(frag, now)

		# allocate socket if needed
		if not udp:
			udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

		# if transmission inhibited, talk about that
		if dryrun:
			sys.stderr.write("DEBUG: udp.sendto(...) inhibited by option.\n")
			continue

		# send it - any exceptions will be caught by the caller
		result = udp.sendto(datagram, (host, port)) # send (UDP)
		sequence += 1 # increment the sequence number
		if verbose:
			status = "OK" if result == len(datagram) else "mismatch"
			sys.stderr.write("DEBUG: udp.sendto(...) returned %d [%s]\n" % (result, status))
		time.sleep(0.5) # short pause to keep network happy


#
#  spot_callback(...)
#
#  Each spot generates a dictionary 'd' with spot information.  This
#  function processes each new spot, and periodically does an upload
#  cycle to send recent spots to the server.
#
#  This is the main worker function for the application.
#
def spot_callback_core(d):
	global update, interval

	# process the spot, if possible
	process_spot(d)

	# only do the upload every 'interval' seconds, at most
	now = time.time()
	elapsed = now - update
	if elapsed < interval:
		return
	
	# run the upload cycle
	upload_cycle(now)


#
#  this is the outer-level callback
#
def spot_callback(d):
	global background
	try:
		spot_callback_core(d)
	except Exception as ex:
		# extract location information and display exception as an error
		exc_type, exc_obj, exc_tb = sys.exc_info()
		fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
		msg = "Caught unexpected: %s (%s) in %s at line %d" % (exc_type, str(ex), fname, exc_tb.tb_lineno)
		if background:
			syslog.syslog(msg)
		else:
			sys.stderr.write(msg + '\n')
			sys.stderr.flush()


#
#  socket_loop(...) - main program loop for socket mode
#
def socket_loop(udp, cb_timeout=60):
	last_event = time.time()
	while True:
		rs = [ ]
		if udp:
			rs.append(udp)
		rs, ws, es = select.select(rs, [], [], 0.500)

		# if a frame was received, convert to ASCII and raise a spot event
		if rs and udp in rs:
			data, who = udp.recvfrom(256)
			if data and who:
				data = data.decode('ascii').strip()
				data = parse_spot(data)
				spot_callback(data)

		# make sure a callback happens now and then, even with no data
		# NOTE: the taillog method does this on its own, so we only
		#       need to do this here for the socket mode
		now = time.time()
		if (now - last_event) >= cb_timeout:
			last_event = now
			spot_callback(None)


#
#  signal_handler(sig, stk_frm)
#
def signal_handler(sig, stk_frm):
	if sig in [ signal.SIGHUP ]:
		return
	if sig in [ signal.SIGINT, signal.SIGTERM ]:
		sys.exit(0)


#
#  main()
#
def main():
	global verbose, mycall, mygrid, path, host, port, client_id
	global interval, background, update, minhits, dryrun

	# bind address (optional)
	uhost = None
	uport = None

	# load settings
	lines = [ ]
	fn = os.path.join(os.path.expanduser('~'), '.ft8reportrc')
	if os.path.isfile(fn):
		with io.open(fn, 'r') as f:
			lines = f.readlines()

		# DEBUG:
		if verbose > 1:
			sys.stderr.write("DEBUG: Read saved state: %s\n" % fn)

	# parse settings
	for line in lines:
		parts = line.split()
		if len(parts) == 2:
			if parts[0] == 'ID':
				client_id = int(parts[1])
			elif parts[0] == 'CALL':
				mycall = parts[1]
			elif parts[0] == 'GRID':
				mygrid = parts[1]

	# read the command line
	optlist, cmdline = getopt.getopt(sys.argv[1:], 'A:a:dhc:g:i:n:rtvz')
	for opt in optlist:
		if opt[0] == '-d':
			verbose += 1
		elif opt[0] == '-h':
			usage()
		elif opt[0] == '-v':
			sys.stdout.write("ft8report version %s\n" % GetModemVersion())
			return 0
		elif opt[0] == '-a':
			path = opt[1]
		elif opt[0] == '-A': # [addr:]port
			uport = opt[1]
			if ':' in uport:
				uport = split(':')
				if len(uport) != 2:
					sys.stderr.write("-A must be either a port number, or a host:port pair.\n")
					sys.exit(1)
				uhost, uport = uport
				if not uport.isdigit():
					sys.stderr.write("-A must be either a port number, or a host:port pair.\n")
					sys.exit(1)
			uport = int(uport)
			if uport <= 0 or uport >= 65536:
				sys.stderr.write("Port number must be between 1 and 65535.\n")
				sys.exit(1)
		elif opt[0] == '-c':
			mycall = opt[1]
		elif opt[0] == '-g':
			mygrid = opt[1]
		elif opt[0] == '-r':
			dryrun = True
		elif opt[0] == '-i':
			client_id = int(opt[1]) & 0x7FFFFFFF
		elif opt[0] == '-n':
			minhits = int(opt[1])
		elif opt[0] == '-z':
			background = True
		elif opt[0] == '-t':
			host = 'pskreporter.info' # test host
			port = 14739              # test UDP port
			interval = 30

	# make sure there is enough information, and not too much
	if not path and not mycall and not mygrid:
		usage();
	if not mycall:
		sys.stderr.write("Must specify receiver call.\n")
		sys.exit(1)
	if not mygrid:
		sys.stderr.write("Must specify grid square.\n")
		sys.exit(1)
	if uport and path:
		sys.stderr.write("Options -a <path> and -A <port> cannot be used together.\n")
		sys.exit(1)
	if not uport and not path:
		sys.stderr.write("Must supply either -a <path> or -A <port>.\n")
		sys.exit(1)
	
	# clean up the input
	mycall = mycall.strip().upper()
	mygrid = mygrid.strip().upper()

	# initialize the timer
	if interval > 60:
		update = time.time() - (interval - 60) # update after 60s, then every 'interval' seconds thereafter

	# DEBUG:
	if verbose:
		sys.stderr.write(
			"DEBUG: Start: call = %s; grid = %s; host = %s; port = %d; id = %d, minhits = %d\n" % (
				mycall, mygrid, host, port, client_id, minhits));
	
	# expand the path, and check that it exists
	if path:
		path = os.path.expanduser(path)
		pids = find_ft8report(path)
		if pids:
			sys.stderr.write("An 'ft8report' is already running for path '%s': " % path)
			for pid in pids:
				sys.stderr.write("[%d] " % pid)
			sys.stderr.write("\nNot starting a new instance, exiting.\n")
			sys.exit(1)
	
	# get the bind information
	if uport:
		if not uhost:
			uhost = '127.0.0.1'
		fail = False
		try:
			ip = socket.gethostbyname(uhost)
			if ip:
				uhost = ip
			else:
				fail = True
		except Exception as ex:
			fail = True
		if fail:
			sys.stderr.write("Could not resolve host: %s\n" % uhost)
			sys.exit(1)
		if verbose:
			sys.stderr.write("DEBUG: UDP host = %s; UDP port = %d\n" % (uhost, uport))

	# allocate the UDP listener
	udp = None
	if uport:
		try:
			udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
			udp.bind((uhost, uport))
		except Exception as ex:
			sys.stderr.write("Could not open listener: %s\n" % str(ex))
			sys.exit(1)

	# fork into background
	if background:
		# cancel verbose messages when running in background
		verbose = False

		# fork into background
		child_pid = os.fork()
		if child_pid > 0: # parent
			sys.exit(0)   # exit
		os.setsid() # set session leader
		os.umask(0) # set umask to zero

		# handle signals when running in background
		signal.signal(signal.SIGHUP, signal_handler)
		signal.signal(signal.SIGINT, signal_handler)
		signal.signal(signal.SIGTERM, signal_handler)

		# set up syslog
		syslog.openlog('ft8report');
		syslog.syslog('Starting in background as pid=%d, path=%s' % (os.getpid(), path))

	# register function to write configuration when exiting
	atexit.register(save_config)

	# run the main loop
	try:
		if path:
			tailall.tailall(path, spot_callback)
		else:
			socket_loop(udp)
	except KeyboardInterrupt as ex:
		sys.exit(0)


#
#  entry point
#
if __name__ == '__main__':
	main()

# EOF
