diff --git a/GA_Detect.py b/GA_Detect.py index 0fe66d0..cc55305 100644 --- a/GA_Detect.py +++ b/GA_Detect.py @@ -1,18 +1,29 @@ """A script to process OpenSky ADS-B data in an attempt to detect go-around events at an airport.""" from traffic.core import Traffic from datetime import timedelta +import importlib import multiprocessing as mp -from OS_Airports import VABB +import metar_parse as MEP import OS_Funcs as OSF import glob - - -def main(start_n, fidder, do_write): +import click + + +@click.command() +@click.option('--top-dir', default='./') +@click.option('--start-n', default=0) +@click.option('--do-write', default=True) +@click.option('--metars-file', default='VABB_METAR') +@click.option('--airport', default='VABB') +@click.option('--n-files-proc', default=55) +@click.option('--pool-proc', default=16) +@click.option('--verbose', default=False) +def main(top_dir, start_n, do_write, metars_file, airport, + n_files_proc, pool_proc, verbose): """The main code for detecting go-arounds. Arguments: start_n -- The index of the first file to read - fidder -- the id of an open file to write log information into do_write -- boolean flag specifying whether to output data to textfile """ @@ -22,7 +33,6 @@ def main(start_n, fidder, do_write): # Of which go-arounds tot_n_ga = 0 - top_dir = '/gf2/eodg/SRP002_PROUD_ADSBREP/GO_AROUNDS/VABB/' # indir stores the opensky data indir = top_dir + 'INDATA/' @@ -41,9 +51,9 @@ def main(start_n, fidder, do_write): odirs = [odir_pl_nm, odir_pl_ga, odir_da_nm, odir_da_ga] # Output filenames for saving data about go-arounds - out_file_ga = 'GA_MET_NEW.csv' + out_file_ga = top_dir + 'GA_MET_NEW.csv' # Output filenames for saving data about non-go-arounds - out_file_noga = 'GA_NOGA_NEW.csv' + out_file_noga = top_dir + 'GA_NOGA_NEW.csv' t_frmt = "%Y/%m/%d %H:%M:%S" @@ -68,10 +78,9 @@ def main(start_n, fidder, do_write): colormap = {'GND': 'black', 'CL': 'green', 'CR': 'blue', 'DE': 'orange', 'LVL': 'purple', 'NA': 'red'} - # Number of files to open in one go - n_files_proc = 55 - - pool_proc = 100 + metars = MEP.get_metars(metars_file, verbose=verbose) + rwy_list = getattr(importlib.import_module(f'OS_Airports.{airport}'), + 'rwy_list') f_data = [] pool = mp.Pool(processes=pool_proc) @@ -80,9 +89,6 @@ def main(start_n, fidder, do_write): print("Processing batch starting with " + str(main_count + 1).zfill(5) + " of " + str(fli_len).zfill(5)) - fidder.write("Processing batch starting with " - + str(main_count + 1).zfill(5) + " of " - + str(fli_len).zfill(5) + '\n') p_list = [] # First we load several files at once @@ -113,11 +119,12 @@ def main(start_n, fidder, do_write): if (flight.stop + timedelta(minutes=5) < end_time): p_list.append(pool.apply_async(OSF.proc_fl, args=(flight, - VABB.rwy_list, + metars, + rwy_list, odirs, colormap, True, - False,))) + verbose,))) else: f_data.append(flight) @@ -196,12 +203,5 @@ def main(start_n, fidder, do_write): nogfid.close() -# Use this to start processing from a given file number. -# Can be helpful if processing fails at some point. -init_num = 0 - -fid = open('/home/proud/Desktop/log.log', 'w') - -main(init_num, fid, False) - -fid.close() +if __name__ == '__main__': + main() diff --git a/OS_Funcs.py b/OS_Funcs.py index 956fa4f..1eda75e 100644 --- a/OS_Funcs.py +++ b/OS_Funcs.py @@ -2,7 +2,6 @@ from scipy.interpolate import UnivariateSpline as UniSpl from traffic.core import Traffic from datetime import timedelta -import metar_parse as MEP import pandas as pd import flightphase as flph @@ -11,10 +10,6 @@ import numpy as np -# Read METARs from disk -metars = MEP.get_metars('/home/proud/Desktop/GoAround_Paper/VABB_METAR') - - def estimate_rwy(df, rwy_list, verbose): """Guess which runway a flight is attempting to land on. @@ -255,11 +250,12 @@ def check_ga(fd, verbose, first_pos=-1): return ga_flag, bpt -def proc_fl(flight, check_rwys, odirs, colormap, do_save, verbose): +def proc_fl(flight, metars, check_rwys, odirs, colormap, do_save, verbose): """Filter, assign phases and determine go-around status for a given flight. Inputs: - A 'traffic' flight object + - A dict of METARS, each as a metobs class - A list storing potential landing runways to check - A 4-element list specifying various output directories: - normal plot output diff --git a/README.md b/README.md index 2acae14..69f95bb 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ Requires: Usage: First you must download aircraft data, which can be done using the `OpenSky_Get_Data` script. You can then point `GA_Detect` at the download location to scan for go-arounds. -This tool is in very early development, so has manual tweaks that would ideally be changeable via a config file or directly via the command line call. The most important of these tweaks are listed below: ### `OpenSky_Get_Data.py`: @@ -34,9 +33,40 @@ python OpenSky_Get_Data.py \ --outdir=INDATA --n-jobs=1 ``` -### In `GA_Detect.py` -The directory structure is set at the beginning of `main()`. You will probably want to adjust this to your own requirements. +### `GA_Detect.py` -`n_files_proc` specifies how many files to process simultaneously. This should be changed to the optimal value for your hardware. +The script that runs the actual go-around events. -`pool_proc` specifies the number of multiprocessing threads to use. I have found that this can be set slightly higher than the number of cores available, as cores are not fully utilised anyway. +Data is read and written based on a top-level directory. The default +is the current working directory, which can be overridden by passing +the `--top-dir` command-line option. + +The file containing the appropriate METARS data can be passed using +the `--metars-file` option. + +The airport can be specified using the `--airport` option. See the +`OS_Airports` subpackage which contains the runway definitions for the +currently supported airports. + +The `--n-files-proc` option specifies how many files to process +simultaneously. This should be changed to the optimal value for your +hardware. + +The `--pool-proc` option specifies the number of multiprocessing +threads to use. I have found that this can be set slightly higher than +the number of cores available, as cores are not fully utilised anyway. + +The `GA_Detect.py` command-line script uses a few defaults. The +defaults are chosen to accommodate the defaults in the data fetching +scripts. As such, these two calls are equivalent: + +```bash +python GA_Detect.py # does the same thing as the next command: +python GA_Detect.py \ + --top-dir=. --metars-file=VABB_METAR --airport=VABB +``` + +## Running tests + +To run the included test suite, first install pytest via `pip install +pytest` and then run `pytest -s tests`. diff --git a/tests/data/INDATA/OS_201908100000_VABB.pkl b/tests/data/INDATA/OS_201908100000_VABB.pkl new file mode 100644 index 0000000..65a47e5 Binary files /dev/null and b/tests/data/INDATA/OS_201908100000_VABB.pkl differ diff --git a/tests/data/VABB_METAR b/tests/data/VABB_METAR new file mode 100644 index 0000000..0992f9f --- /dev/null +++ b/tests/data/VABB_METAR @@ -0,0 +1,5 @@ +VABB,2019-08-10 00:00,VABB 100000Z 23010G20KT 2500 HZ SCT018 FEW025TCU BKN090 28/25 Q1000 NOSIG +VABB,2019-08-10 00:30,VABB 100030Z 24010G20KT 2500 HZ SCT018 FEW025TCU BKN090 28/25 Q1000 TEMPO 1500 RA +VABB,2019-08-10 01:00,VABB 100100Z 25011G21KT 2500 HZ SCT012 SCT018 BKN090 28/25 Q1000 TEMPO 1500 RA +VABB,2019-08-10 01:30,VABB 100130Z 24014G24KT 2500 HZ FEW012 SCT018 BKN090 28/25 Q1001 NOSIG +VABB,2019-08-10 02:00,VABB 100200Z 24009G19KT 2500 HZ FEW012 SCT018 BKN090 28/26 Q1001 NOSIG diff --git a/tests/data/expected/GA_MET_NEW.csv b/tests/data/expected/GA_MET_NEW.csv new file mode 100644 index 0000000..052f52f --- /dev/null +++ b/tests/data/expected/GA_MET_NEW.csv @@ -0,0 +1,2 @@ +ICAO24, Callsign, GA_Time, L_Time, Runway, Heading, Alt, Lat, Lon, gapt, rocvar, hdgvar, latvar, lonvar, gspvar, Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base, CB, Vis, Pressure +04c147,KQA204,2019/08/10 00:21:24,2019/08/10 00:19:25,27,263.9909940425055,1347.6201812316162,19.08824157714844,72.8559820992606,370,1.3554425701202464,0.0027863795136474465,3.5391823044835526e-06,0.00021195186832100995,0.03189928263824995,28.0,25.0,10.0,20.0,240.0,1800.0,0,2500.0,1000.0 diff --git a/tests/data/expected/GA_NOGA_NEW.csv b/tests/data/expected/GA_NOGA_NEW.csv new file mode 100644 index 0000000..3f1bc1b --- /dev/null +++ b/tests/data/expected/GA_NOGA_NEW.csv @@ -0,0 +1,7 @@ +ICAO24, Callsign, GA_Time, L_Time, Runway, gapt, rocvar, hdgvar, latvar, lonvar, gspvar, Temp, Dewp, Wind_Spd, Wind_Gust, Wind_Dir,Cld_Base, CB, Vis, Pressure +8005ee,AIC317,2019/08/10 00:26:12,2019/08/10 00:16:54,27,0,1.5964404850235485,0.0006783574870077095,2.4668402388350134e-06,0.0001930544997218994,0.11175369742826806,28.0,25.0,10.0,20.0,240.0,1800.0,0,2500.0,1000.0 +800732,AIC124,2019/08/10 00:05:45,2019/08/10 00:00:01,27,0,1.4635712542703654,0.009514184629520878,2.7480109772978474e-06,0.00019246633015406858,0.026726124191242435,28.0,25.0,10.0,20.0,230.0,1800.0,0,2500.0,1000.0 +800bb6,IGO218,2019/08/10 00:45:44,2019/08/10 00:38:25,27,0,0.7520124217363429,0.00612862439723081,2.324620750136083e-06,0.00016878190829216932,0.04790138895453693,28.0,25.0,11.0,21.0,250.0,1200.0,0,2500.0,1000.0 +800c69,GOW030,2019/08/10 00:08:06,2019/08/10 00:00:01,27,0,0.8958133916464194,0.00029074779492562153,2.0333079803969063e-06,0.0001726147052984897,0.036253078686998626,28.0,25.0,10.0,20.0,230.0,1800.0,0,2500.0,1000.0 +800c71,IGO456,2019/08/10 00:48:02,2019/08/10 00:41:43,27,0,0.950105112352263,0.011971063410794125,2.681003129011264e-06,0.00020661348328495407,0.09270944570168702,28.0,25.0,11.0,21.0,250.0,1200.0,0,2500.0,1000.0 +800dac,IGO235,2019/08/10 00:32:31,2019/08/10 00:24:02,27,0,0.8887419529611325,0.01015078239663644,2.1995557883975666e-06,0.00015366319326703428,0.05454545454545455,28.0,25.0,10.0,20.0,240.0,1800.0,0,2500.0,1000.0 diff --git a/tests/test_GA_detect.py b/tests/test_GA_detect.py new file mode 100644 index 0000000..ff6ab15 --- /dev/null +++ b/tests/test_GA_detect.py @@ -0,0 +1,42 @@ +import os +import shutil +import sys + +from click.testing import CliRunner +import pytest + +sys.path.insert(0, '.') + + +@pytest.fixture +def data_dir(): + return os.path.join(os.path.dirname(__file__), 'data') + + +@pytest.fixture +def data_dir_copy(data_dir, tmpdir): + dest = str(tmpdir) + '/' + os.path.basename(data_dir) + shutil.copytree(data_dir, dest) + return dest + + +class TestMain: + @pytest.fixture + def vabb(self, data_dir_copy): + from GA_Detect import main + runner = CliRunner() + result = runner.invoke(main, [ + '--top-dir', data_dir_copy + '/', + '--metars-file', os.path.join(data_dir_copy, 'VABB_METAR'), + '--pool-proc', '1', + ]) + assert result.exit_code == 0 + return result + + def test_csv(self, vabb, data_dir, data_dir_copy): + for fname in ['GA_MET_NEW.csv', 'GA_NOGA_NEW.csv']: + with open(os.path.join(data_dir, 'expected', fname)) as f: + csv_expected = f.read() + with open(os.path.join(data_dir_copy, fname)) as f: + csv_got = f.read() + assert csv_expected == csv_got