Generated: Sat Oct 24 16:35:19 2020 from getcfxml02.pl 2018/03/06 19.2 KB. text copy
#!/usr/bin/perl -w # NAME: getcfxml02.pl # AIM: Fetch the crossfeed flights.xml, and keep some flight stats, using XML:Simple # 2018-03-06 - Review and make some changes, add seconds, verbosity, out dir, etc... use strict; use warnings; use File::Basename; # split path ($name,$dir,$ext) = fileparse($file [, qr/\.[^.]*/] ) use File::stat; use XML::Simple; use Data::Dumper; use LWP::Simple; use JSON; use Cwd; my $os = $^O; my $perl_dir = '/home/geoff/bin'; my $PATH_SEP = '/'; my $temp_dir = '/tmp'; if ($os =~ /win/i) { $perl_dir = 'C:\GTools\perl'; $temp_dir = $perl_dir; $PATH_SEP = "\\"; } unshift(@INC, $perl_dir); require 'lib_utils.pl' or die "Unable to load 'lib_utils.pl' Check paths in \@INC...\n"; # log file stuff our ($LF); my $pgmname = $0; if ($pgmname =~ /(\\|\/)/) { my @tmpsp = split(/(\\|\/)/,$pgmname); $pgmname = $tmpsp[-1]; } my $outfile = $temp_dir.$PATH_SEP."temp.$pgmname.txt"; open_log($outfile); # user variables my $VERS = "0.0.2 2018-03-06"; #my $VERS = "0.0.1 2013-04-30"; my $load_log = 0; my $in_file = ''; my $verbosity = 1; my $out_dir = $perl_dir; my $json_dir = $out_dir.$PATH_SEP.'json'; # use <root>/json as output my $out_file = $json_dir.$PATH_SEP.'usage01.json'; my $out_file2 = $json_dir.$PATH_SEP.'usage01.txt'; my $url = 'http://crossfeed.freeflightsim.org/flights.xml'; ###my $url = 'http://crossfeed.fgx.ch/flights.xml'; my $oldest_time = time(); my $sleep_secs = 10; # ### DEBUG ### my $debug_on = 0; #my $def_file = 'def_file'; my $def_file = $temp_dir.$PATH_SEP."tempcfxml.txt";; my $def_out = $temp_dir.$PATH_SEP.'tempdef.json'; ### program variables my @warnings = (); my $cwd = cwd(); my $max_cs = 0; my $max_mod = 0; my %model_hash = (); my %callsign_hash = (); sub write_ref_hashes($); sub VERB1() { return $verbosity >= 1; } sub VERB2() { return $verbosity >= 2; } sub VERB5() { return $verbosity >= 5; } sub VERB9() { return $verbosity >= 9; } sub show_warnings($) { my ($val) = @_; if (@warnings) { prt( "\nGot ".scalar @warnings." WARNINGS...\n" ); foreach my $itm (@warnings) { prt("$itm\n"); } prt("\n"); } else { prt( "\nNo warnings issued.\n\n" ) if (VERB9()); } } sub pgm_exit($$) { my ($val,$msg) = @_; if (length($msg)) { $msg .= "\n" if (!($msg =~ /\n$/)); prt($msg); } show_warnings($val); close_log($outfile,$load_log); exit($val); } sub prtw($) { my ($tx) = shift; $tx =~ s/\n$//; prt("$tx\n"); push(@warnings,$tx); } # RENAME A FILE TO .1, .2, .3... sub rename_2_nxt_num { my ($fil) = shift; my $ret = 0; # assume NO SUCH FILE if ( -f $fil ) { # is there? $ret = 1; my $nfil = $fil.".$ret"; while (-f $nfil) { $ret++; $nfil = $fil.".$ret"; } rename $fil, $nfil; } return $ret; } sub get_oldest_time_in_dir($) { my $dir = shift; if (!opendir(DIR,$dir)) { return time(); } my @files = readdir(DIR); closedir(DIR); ut_fix_directory(\$dir); my ($file,$ff,$sb); my $tm = time(); foreach $file (@files) { next if ($file eq '.'); next if ($file eq '..'); $ff = $dir.$file; next if (-d $ff); # skip directories if ($sb = stat($ff)) { $tm = $sb->mtime if ($sb->mtime < $tm); } } return $tm; } #Processing 74 lines, from [tempusage.json]... #Dump of perl scalar... #$VAR1 = { # 'cs_cnt' => 41, # 'models' => [ # { # 'usr_cnt' => 1, # 'model' => '717-200', # 'users' => [ # 'dedokun' # ] # }, # ], # 'success' => bless( do{\(my $o = 1)}, 'JSON::XS::Boolean' ), # 'callsigns' => [ # { # 'models' => [ # '777-200' # ], # 'callsign' => 'ANA108', # 'mod_cnt' => 1 # }, # ], # 'updated' => '2013/04/30 12:23:45 UTC', # 'mod_cnt' => 28, # 'generator' => 'xmlsimple02.pl' # }; # #End DUMP sub reload_previous_json($) { my $inf = shift; if (! open INF, "<$inf") { my ($name,$dir) = fileparse($inf); if (-d $dir) { prtw("WARNING: Unable to open file [$inf]. Assume none...\n"); } else { prt("WARNING: Unable to open file [$inf]...\n"); pgm_exit(1,"ERROR: Unable to stat dir [$dir]\n"); } return; } my @lines = <INF>; close INF; my $lncnt = scalar @lines; prt("Processing $lncnt lines, from [$inf]...\n"); my $line = join(" ",@lines); my $json = JSON->new->allow_nonref; my $rh = $json->decode( $line ); ###my $tmp = Dumper($rh); my ($ra,$cnt,$mc,$cc,$mh,$mod,$rua,$user,$refha,$cs,$len); $mc = 0; $cc = 0; if (defined ${$rh}{'mod_cnt'}) { $mc = ${$rh}{'mod_cnt'}; } if (defined ${$rh}{'cs_cnt'}) { $cc = ${$rh}{'cs_cnt'}; } if (defined ${$rh}{'models'}) { $ra = ${$rh}{'models'}; $cnt = scalar @{$ra}; prt("Got $cnt model hashes...($mc)\n"); foreach $mh (@{$ra}) { # 'usr_cnt' => 1, # 'model' => '717-200', # 'users' => ['dedokun'] if ((defined ${$mh}{'model'})&& (defined ${$mh}{'users'})) { $mod = ${$mh}{'model'}; $len = length($mod); $max_mod = $len if ($len > $max_mod); $rua = ${$mh}{'users'}; $model_hash{$mod} = {} if (!defined $model_hash{$mod}); $refha = $model_hash{$mod}; foreach $cs (@{$rua}) { if (defined ${$refha}{$cs}) { ${$refha}{$cs}++; } else { ${$refha}{$cs} = 1; $len = length($cs); $max_cs = $len if ($len > $max_cs); } } } else { prt(Dumper($mh)); pmg_exit(2,"ERROR: 'model' or 'users' NOT defined!\n"); } } } if (defined ${$rh}{'callsigns'}) { $ra = ${$rh}{'callsigns'}; $cnt = scalar @{$ra}; prt("Got $cnt callsign hashes...($cc)\n"); # 'models' => ['777-200'], # 'callsign' => 'ANA108', # 'mod_cnt' => 1 foreach $mh (@{$ra}) { if ((defined ${$mh}{'callsign'})&& (defined ${$mh}{'models'})) { $cs = ${$mh}{'callsign'}; $rua = ${$mh}{'models'}; $callsign_hash{$cs} = {} if (!defined $callsign_hash{$cs}); $refha = $callsign_hash{$cs}; foreach $user (@{$rua}) { if (defined ${$refha}{$user}) { ${$refha}{$user}++; } else { ${$refha}{$user} = 1; } } } else { prt(Dumper($mh)); pmg_exit(2,"ERROR: 'callsign' or 'users' NOT defined!\n"); } } } rename_2_nxt_num($inf); # get it out of the way for the next write_ref_hashes($def_out) if (VERB9()); } ################################################ ### json format # Characters '"', '\', and '/' are escaped with '\' sub get_json($) { my $txt = shift; my $len = length($txt); my ($i,$ch); my $json = ''; for ($i = 0; $i < $len; $i++) { $ch = substr($txt,$i,1); $json .= "\\" if (($ch eq '"')||($ch eq "\\")||($ch eq '/')) ; $json .= $ch; } return $json; } sub mycmp_nc_sort { return -1 if (lc($a) lt lc($b)); return 1 if (lc($a) gt lc($b)); return 0; } sub mycmp_ascend_n1 { return -1 if (${$a}[1] < ${$b}[1]); return 1 if (${$a}[1] > ${$b}[1]); return 0; } sub mycmp_decend_n1 { return -1 if (${$a}[1] > ${$b}[1]); return 1 if (${$a}[1] < ${$b}[1]); return 0; } sub get_hash_list($) { my $rh = shift; my $msg = ''; my ($key,$val); my @arr = sort mycmp_nc_sort keys(%{$rh}); foreach $key (@arr) { $val = ${$rh}{$key}; $msg .= "," if (length($msg)); $msg .= "$key=$val"; } return $msg; } sub out_usage_list($$) { my ($ra,$msg) = @_; # \@usage,$msg my ($val,$mod,$cnt,$tcnt,@arr,$tmp,$acnt); # combine some models my %h = (); $tcnt = 0; my $mod_777 = '777-*'; my $mod_atc = 'ATC-*'; my %mod_hash = (); my %atc_hash = (); foreach $val (@{$ra}) { $tcnt++; $mod = ${$val}[0]; $cnt = ${$val}[1]; if ($mod =~ /^777-/) { if (defined $mod_hash{$mod}) { $mod_hash{$mod} += $cnt; } else { $mod_hash{$mod} = $cnt; } $mod = $mod_777; # '777-*'; } elsif (($mod =~ /atc/i)||($mod =~ /OpenRadar/i)) { if (defined $atc_hash{$mod}) { $atc_hash{$mod} += $cnt; } else { $atc_hash{$mod} = $cnt; } $mod = $mod_atc; # 'ATC-*'; } if (defined $h{$mod}) { $h{$mod} += $cnt; } else { $h{$mod} = $cnt; } } my @usage = (); my $total = 0; @arr = sort mycmp_nc_sort keys(%h); foreach $mod (@arr) { $cnt = $h{$mod}; push(@usage,[$mod,$cnt]); $total += $cnt; } @usage = sort mycmp_decend_n1 @usage; $cnt = scalar @usage; my $curr = time(); my $diff = $curr - $oldest_time; my $rt = secs_HHMMSS($diff); my $txt = "# Usage list, from ".lu_get_YYYYMMDD_hhmmss_UTC($oldest_time)." UTC, now at ".lu_get_YYYYMMDD_hhmmss_UTC($curr)." UTC\n"; $txt .= "# Total of $cnt model groups, $tcnt total, with $total callsigns, over $rt\n"; foreach $val (@usage) { $mod = ${$val}[0]; $cnt = ${$val}[1]; $tmp = ''; if ($mod eq $mod_777) { $tmp = get_hash_list(\%mod_hash); } elsif ($mod eq $mod_atc) { $tmp = get_hash_list(\%atc_hash); } $mod .= ' ' while (length($mod) < $max_mod); $acnt = sprintf("%3d",$cnt); $txt .= "$mod $acnt $tmp\n"; } $txt .= "# eof $out_file2\n"; write2file($txt,$out_file2); prt("$rt $msg\n") if (VERB1()); } sub write_ref_hashes($) { my $out = shift; my ($key,$val,$key2,$val2,$rh,$json,@arr,$cnt,@arr2,$cnt2,$msg); $rh = \%model_hash; my @usage = (); # per model @arr2 = sort mycmp_nc_sort keys(%{$rh}); $cnt = scalar @arr2; $json = '{"success":true,"generator":"'.$pgmname.'","updated":"'.lu_get_YYYYMMDD_hhmmss_UTC(time()).' UTC",'."\n"; $json .= '"mod_cnt":'.$cnt; $json .= ',"models":['."\n"; $msg = "Model count $cnt"; $cnt2 = 0; foreach $key (@arr2) { $val = ${$rh}{$key}; @arr = sort keys(%{$val}); $cnt = scalar @arr; $json .= ",\n" if ($cnt2); $json .= '{"model":"'.get_json($key).'","usr_cnt":'.$cnt.',"users":['; push(@usage, [$key,$cnt]); $cnt = 0; foreach $key2 (@arr) { $val2 = ${$val}{$key2}; # get count by pilot $json .= ',' if ($cnt); $json .= '"'.get_json($key2).'"'; $cnt++; } $json .= ']}'; $cnt2++; } $json .= "\n"; $json .= '],'."\n"; # per callsign $rh = \%callsign_hash; @arr2 = sort mycmp_nc_sort keys(%{$rh}); $cnt = scalar @arr2; $json .= '"cs_cnt":'.$cnt; $json .= ',"callsigns":['."\n"; $msg .= ", callsign count $cnt"; $cnt2 = 0; foreach $key (@arr2) { $val = ${$rh}{$key}; @arr = sort keys(%{$val}); $cnt = scalar @arr; $json .= ",\n" if ($cnt2); $json .= '{"callsign":"'.get_json($key).'","mod_cnt":'.$cnt.',"models":['; $cnt = 0; foreach $key2 (@arr) { $val2 = ${$val}{$key2}; # get count by pilot $json .= ',' if ($cnt); $json .= '"'.get_json($key2).'"'; $cnt++; } $json .= ']}'; $cnt2++; } $json .= "\n"; $json .= ']}'."\n"; prt($json) if (VERB9()); write2file($json,$out); $msg .= ", to [$out]"; ##prt("$msg, to [$out]\n"); # out a USAGE list out_usage_list(\@usage,$msg); } sub process_in_file($) { my ($inf) = @_; prt("Processing [$inf]...\n") if (VERB9()); my $xml = XMLin($inf); if (defined ${$xml}{'marker'}) { my $ra = ${$xml}{'marker'}; #prt("Dump of ref array...\n"); #prt(Dumper($ra)); my ($rh,$cs,$mod,$refha,$len); prt("Dump of each hash...or callsign and model strings\n") if (VERB9()); #'alt' => '102', #'callsign' => 'gagaga', #'lat' => '36.714629', #'model' => 'v22', #'heading' => '8', #'spd_kt' => '0', #'lng' => '126.492990', #'server_ip' => '203.251.5.253' foreach $rh (@{$ra}) { if ( (defined ${$rh}{'callsign'}) && (defined ${$rh}{'model'}) ) { $cs = ${$rh}{'callsign'}; $mod = ${$rh}{'model'}; prt("$mod $cs\n") if (VERB9()); $model_hash{$mod} = {} if (!defined $model_hash{$mod}); $refha = $model_hash{$mod}; if (defined ${$refha}{$cs}) { ${$refha}{$cs}++; } else { ${$refha}{$cs} = 1; $len = length($cs); $max_cs = $len if ($len > $max_cs); } $callsign_hash{$cs} = {} if (!defined $callsign_hash{$cs}); $refha = $callsign_hash{$cs}; if (defined ${$refha}{$mod}) { ${$refha}{$mod}++; } else { ${$refha}{$mod} = 1; $len = length($mod); $max_mod = $len if ($len > $max_mod); } } else { prt(Dumper($rh)); } } } else { prt("Dump of WHOLE\n"); prt(Dumper($xml)); } #my $xml2 = XMLin($inf, KeyAttr => {marker => 'model'}); #prt(Dumper($xml2)); #$load_log = 1; write_ref_hashes($out_file); } sub get_crossfeed() { my $cf = get($url); process_in_file($cf); } ######################################### ### MAIN ### parse_args(@ARGV); ### pgm_exit(1,""); $oldest_time = get_oldest_time_in_dir($json_dir); reload_previous_json($out_file); while (1) { get_crossfeed(); sleep($sleep_secs); # default was 10 } #process_in_file($in_file); pgm_exit(0,""); ######################################## sub need_arg { my ($arg,@av) = @_; pgm_exit(1,"ERROR: [$arg] must have a following argument!\n") if (!@av); } sub parse_args { my (@av) = @_; my ($arg,$sarg); while (@av) { $arg = $av[0]; if ($arg =~ /^-/) { $sarg = substr($arg,1); $sarg = substr($sarg,1) while ($sarg =~ /^-/); if (($sarg =~ /^h/i)||($sarg eq '?')) { give_help(); pgm_exit(0," Help exit(0)"); } elsif ($sarg =~ /^v/) { if ($sarg =~ /^v.*(\d+)$/) { $verbosity = $1; } else { while ($sarg =~ /^v/) { $verbosity++; $sarg = substr($sarg,1); } } prt("Verbosity = $verbosity\n") if (VERB1()); #} elsif ($sarg =~ /^l/) { # if ($sarg =~ /^ll/) { # $load_log = 2; # } else { # $load_log = 1; # } # prt("Set to load log at end. ($load_log)\n") if (VERB1()); } elsif ($sarg =~ /^o/) { need_arg(@av); shift @av; $sarg = $av[0]; $out_dir = $sarg; if (-d $out_dir) { $json_dir = $out_dir; # use user dir as output $out_file = $json_dir.$PATH_SEP.'usage01.json'; $out_file2 = $json_dir.$PATH_SEP.'usage01.txt'; prt("Set output dir to [$out_dir].\n") if (VERB1()); } else { prt("Error: Can NOT find dir '$out_dir'!\n"); pgm_exit(1, "Output dir MUST exist. Will NOT be created`\n"); } } elsif ($sarg =~ /^s/) { need_arg(@av); shift @av; $sarg = $av[0]; if ($sarg =~ /^\d+$/) { if ($sarg == 0) { pgm_exit(1,"ERROR: Invalid argument [$arg $sarg]! Min is 1\n"); } $sleep_secs = $sarg; } else { prt("Seconds MUST be an integer, not $sarg\n"); pgm_exit(1,"ERROR: Invalid argument [$arg $sarg]! Try -?\n"); } } else { pgm_exit(1,"ERROR: Invalid argument [$arg]! Try -?\n"); } } else { pgm_exit(1,"ERROR: Invalid argument [$arg]! Try -?\n"); #$in_file = $arg; #prt("Set input to [$in_file]\n") if (VERB1()); } shift @av; } #if ($debug_on) { # prtw("WARNING: DEBUG is ON!\n"); # if ((length($in_file) == 0) && $debug_on) { # $in_file = $def_file; # prt("Set DEFAULT input to [$in_file]\n"); # } #} #if (length($in_file) == 0) { # pgm_exit(1,"ERROR: No input files found in command!\n"); #} #if (! -f $in_file) { # pgm_exit(1,"ERROR: Unable to find in file [$in_file]! Check name, location...\n"); #} #$verbosity = 9; } sub give_help { prt("$pgmname: version $VERS\n"); prt("Usage: $pgmname [options]\n"); prt("Options:\n"); prt(" --help (-h or -?) = This help, and exit 0.\n"); prt(" --verb[n] (-v) = Bump [or set] verbosity. def=$verbosity\n"); #prt(" --load (-l) = Load LOG at end. ($outfile)\n"); prt(" --out <dir> (-o) = Write outputs to this dir. (def=$json_dir)\n"); prt(" --secs <sec> (-s) = An integer num of seconds to sleep between fetches. (def=$sleep_secs)\n"); prt("\n"); prt(" If not 'help' then will commence to fetch the crossfeed xml each $sleep_secs secs\n"); prt(" from this $url,\n"); prt(" and process it to extract aircraft and pilots, and keep an updated\n"); prt(" list output to $out_file2 until stopped, ctrl+c...\n"); prt(" The list with more details is also written to $out_file.\n"); prt(" In other words gathers MP aircraft usage using the crossfeed xml.\n"); } # eof - getcfxml02.pl