Update.php

<?php
class Update extends CI_Controller {

	/*
		Controls Updating Elements of Cloudlog
		Functions:
			dxcc - imports the latest clublog cty.xml data
			lotw_users - imports lotw users
	*/

	public function index()
	{
	    $data['page_title'] = "Updates";
	    $this->load->view('interface_assets/header', $data);
	    $this->load->view('update/index');
	    $this->load->view('interface_assets/footer');

	}

    /*
     * Create a path to a file in the updates folder, respecting the datadir
     * configuration option.
     */
    private function make_update_path($path) {
        $path = "updates/" . $path;
        $datadir = $this->config->item('datadir');
        if(!$datadir) {
            return $path;
        }
        return $datadir . "/" . $path;
    }

    /*
     * Load the dxcc entities
     */
	public function dxcc_entities() {
		// Load Database connectors
		$this->load->model('dxcc_entities');

		// Load the cty file
		$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
		
		//$xml_data->entities->entity->count();

        $count = 0;
		foreach ($xml_data->entities->entity as $entity) {
			$startinfo = strtotime($entity->start);
            $endinfo = strtotime($entity->end);
            
            $start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
            $end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
        
            if(!$entity->cqz) {
                $data = array(
                    'prefix' => (string) $entity->call,
                    'name' =>  (string) $entity->entity,
                );
            } else {
                $data = array(
                    'adif' => (int) $entity->adif,
                    'name' =>  (string) $entity->name,
                    'prefix' => (string)  $entity->prefix,
                    'ituz' => (float) $entity->ituz,
                    'cqz' => (int) $entity->cqz,
                    'cont' => (string) $entity->cont,
                    'long' => (float) $entity->long,
                    'lat' => (float) $entity->lat,
                	'start' => $start_date,
                    'end' => $end_date,
                );	
            }
        
            $this->db->insert('dxcc_entities', $data); 
            $count += 1;
            if ($count % 10  == 0)
                $this->update_status();
		}

        $this->update_status();
	    return $count;	
	}

    /*
     * Load the dxcc exceptions
     */
	public function dxcc_exceptions() {
		// Load Database connectors
		$this->load->model('dxcc_exceptions');
		// Load the cty file
		$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
		
        $count = 0;
		foreach ($xml_data->exceptions->exception as $record) {
			$startinfo = strtotime($record->start);
            $endinfo = strtotime($record->end);
            
            $start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
            $end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;

            $data = array(
            	'record' => (int) $record->attributes()->record,
            	'call' => (string) $record->call,
            	'entity' =>  (string) $record->entity,
                'adif' => (int) $record->adif,
                'cqz' => (int) $record->cqz,
                'cont' => (string) $record->cont,
                'long' => (float) $record->long,
                'lat' => (float) $record->lat,
                'start' => $start_date,
                'end' => $end_date,
            );
       
            $this->db->insert('dxcc_exceptions', $data); 
            $count += 1;
            if ($count % 10  == 0)
                $this->update_status();
		}

        $this->update_status();
	    return $count;	
	}

    /*
     * Load the dxcc prefixes
     */
	public function dxcc_prefixes() {
		// Load Database connectors
		$this->load->model('dxcc_prefixes');
		// Load the cty file
		$xml_data = simplexml_load_file($this->make_update_path("cty.xml"));
		
        $count = 0;
		foreach ($xml_data->prefixes->prefix as $record) {
			$startinfo = strtotime($record->start);
            $endinfo = strtotime($record->end);
            
            $start_date = ($startinfo) ? date('Y-m-d H:i:s',$startinfo) : null;
            $end_date = ($endinfo) ? date('Y-m-d H:i:s',$endinfo) : null;
            
            $data = array(
            	'record' => (int) $record->attributes()->record,
            	'call' => (string) $record->call,
            	'entity' =>  (string) $record->entity,
                'adif' => (int) $record->adif,
                'cqz' => (int) $record->cqz,
                'cont' => (string) $record->cont,
                'long' => (float) $record->long,
                'lat' => (float) $record->lat,
                'start' => $start_date,
                'end' => $end_date,
            );
       
            $this->db->insert('dxcc_prefixes', $data); 
            $count += 1;
            if ($count % 10  == 0)
                $this->update_status();
		}

		//print("$count prefixes processed");
        $this->update_status();
	    return $count;	
	}

	// Updates the DXCC & Exceptions from the Club Log Cty.xml file.
	public function dxcc() {
	    $this->update_status("Downloading file");

	    // give it 10 minutes...
	    set_time_limit(600);
	
		// Load Migration data if any.
		$this->load->library('migration');
		$this->fix_migrations();
		$this->migration->latest();

		// Download latest file.
		$url = "https://cdn.clublog.org/cty.php?api=a11c3235cd74b88212ce726857056939d52372bd";
		
		$gz = gzopen($url, 'r');
		if ($gz === FALSE) {
            // If the download from clublog.org fails, try cloudlog.org CDN.
            $url = "https://cdn.cloudlog.org/clublogxml.gz";
            $gz = gzopen($url, 'r');

            // Log failure to log file
            log_message('info', 'Failed to download cty.xml from clublog.org, trying cloudlog.org CDN');

            if ($gz === FALSE) {
                $this->update_status("FAILED: Could not download from clublog.org or cloudlog.org");
                log_message('error', 'FAILED: Could not download exceptions from clublog.org or cloudlog.org');
                return;
            }
		}

		$data = "";
		while (!gzeof($gz)) {
		  $data .= gzgetc($gz);
		}
		gzclose($gz);

		if (file_put_contents($this->make_update_path("cty.xml"), $data) === FALSE) {
			$this->update_status("FAILED: Could not write to cty.xml file");
			return;
		}
	
	    // Clear the tables, ready for new data
		$this->db->empty_table("dxcc_entities");
		$this->db->empty_table("dxcc_exceptions");
		$this->db->empty_table("dxcc_prefixes");
		$this->update_status();

	    // Parse the three sections of the file and update the tables
	    $this->db->trans_start();
		$this->dxcc_entities();
		$this->dxcc_exceptions();
		$this->dxcc_prefixes();
		$this->db->trans_complete();

		$this->update_status("DONE");
	}

	public function get_status() {
		$status_file = $this->make_update_path("status.html");
		if (file_exists($status_file)) {
			$content = file_get_contents($status_file);
			echo $content;
		} else {
			echo "No status available";
		}
	}

	public function update_status($done=""){

        if ($done != "Downloading file"){
            // Check that everything is done?
            if ($done == ""){
                $done = "Updating...";
            }
            $html = $done."<br/>";
            $html .= "Dxcc Entities: ".$this->db->count_all('dxcc_entities')."<br/>";
            $html .= "Dxcc Exceptions: ".$this->db->count_all('dxcc_exceptions')."<br/>";
            $html .= "Dxcc Prefixes: ".$this->db->count_all('dxcc_prefixes')."<br/>";
        }else{
            $html = $done."....<br/>";
        }

        $status_file = $this->make_update_path("status.html");
        if (file_put_contents($status_file, $html) === FALSE) {
            log_message('error', 'Failed to write status file: ' . $status_file);
            // Try to create the directory if it doesn't exist
            $dir = dirname($status_file);
            if (!is_dir($dir)) {
                mkdir($dir, 0755, true);
                file_put_contents($status_file, $html);
            }
        }
	}


	private function fix_migrations(){
        $res = $this->db->query("select version from migrations");
        if ($res->num_rows() >0){
            $row = $res->row();
            $version = $row->version;

            if ($version < 7){
                $this->db->query("update migrations set version=7");
            }
        }
	}
	
	public function check_missing_dxcc($all = false){
	    $this->load->model('logbook_model');
        $this->logbook_model->check_missing_dxcc_id($all);

	}
	
	public function check_missing_continent() {
		$this->load->model('logbook_model');
		$this->logbook_model->check_missing_continent();
	}

	public function update_distances() {
		$this->load->model('logbook_model');
		$this->logbook_model->update_distances();
	}

	public function check_missing_grid($all = false){
	    $this->load->model('logbook_model');
        $this->logbook_model->check_missing_grid_id($all);
	}

    public function update_clublog_scp() {
        $strFile = $this->make_update_path("clublog_scp.txt");
        $url = "https://cdn.clublog.org/clublog.scp.gz";
        set_time_limit(300);
        echo "Downloading Club Log SCP file...<br>";
        $gz = gzopen($url, 'r');
        if ($gz)
        {
            $data = "";
            while (!gzeof($gz)) {
                $data .= gzgetc($gz);
            }
            gzclose($gz);
            if (file_put_contents($strFile, $data) !== FALSE)
            {
                $nCount = count(file($strFile));
                if ($nCount > 0)
                {
                    echo "DONE: " . number_format($nCount) . " callsigns loaded";
                } else {
                    echo "FAILED: Empty file";
                }
            } else {
                echo "FAILED: Could not write to Club Log SCP file";
            }
        } else {
            echo "FAILED: Could not connect to Club Log";
        }
    }

    public function download_lotw_users() {
        $contents = file_get_contents('https://lotw.arrl.org/lotw-user-activity.csv', true);

        if($contents === FALSE) { 
            echo "Something went wrong with fetching the LoTW users file.";
        } else {
            $file = './updates/lotw_users.csv';

            if (file_put_contents($file, $contents) !== FALSE) {     // Save our content to the file.
                echo "LoTW User Data Saved.";
            } else {
                echo "FAILED: Could not write to LoTW users file";
            }
        }

    }

    public function lotw_users() {
        $mtime = microtime(); 
        $mtime = explode(" ",$mtime); 
        $mtime = $mtime[1] + $mtime[0]; 
        $starttime = $mtime; 

        $file = 'https://lotw.arrl.org/lotw-user-activity.csv';

        // First, try to download and validate the file
        $context = stream_context_create([
            'http' => [
                'timeout' => 60, // 60 second timeout
                'user_agent' => 'Cloudlog LoTW User Updater'
            ]
        ]);

        $handle = fopen($file, "r", false, $context);
        if ($handle === FALSE) {
            echo "FAILED: Could not connect to LoTW server";
            log_message('error', 'Failed to connect to LoTW user activity CSV');
            return;
        }

        // Read and validate the first few lines to ensure we have valid data
        $line_count = 0;
        $sample_data = array();
        while ($line_count < 5 && ($line = fgetcsv($handle, 1000, ",")) !== FALSE) {
            if ($line[0]) { // Skip empty lines
                $sample_data[] = $line;
                $line_count++;
            }
        }

        // Validate we have enough sample data
        if ($line_count < 2) {
            fclose($handle);
            echo "FAILED: LoTW file appears to be empty or invalid";
            log_message('error', 'LoTW user activity CSV appears empty or invalid');
            return;
        }

        // Reset file pointer to beginning
        fclose($handle);
        $handle = fopen($file, "r", false, $context);
        if ($handle === FALSE) {
            echo "FAILED: Could not reopen LoTW file";
            return;
        }

        // Only truncate table AFTER we've validated the remote file
        $this->db->query("TRUNCATE TABLE lotw_users"); 
        
        $i = 0; // raw rows read
        $batch_count = 0;
        // Use a map to deduplicate by callsign and keep the latest timestamp
        $lotw_map = array();
        $batch_size = 500; // Smaller batch size for better performance and memory usage
        
        // Skip CSV header row
        $data = fgetcsv($handle,1000,",");
        
        while (($data = fgetcsv($handle,1000,",")) !== FALSE) {
            if ($data[0] && isset($data[1]) && isset($data[2])) {
                // Normalize callsign to uppercase before validation
                $callsign = strtoupper($data[0]);
                // Validate callsign format (basic check)
                if (preg_match('/^[A-Z0-9\/]+$/', $callsign)) {
                    $i++;
                    // Compose timestamp string and compare; keep the latest per callsign
                    $ts = $data[1] . ' ' . $data[2];
                    // If we haven't seen this callsign, or this row is newer, store it
                    if (!isset($lotw_map[$callsign]) || strtotime($ts) > strtotime($lotw_map[$callsign])) {
                        $lotw_map[$callsign] = $ts;
                    }
                }
            }
        }
        fclose($handle);
        
        // Insert deduplicated records in batches
        if (!empty($lotw_map)) {
            $lotwdata = array();
            foreach ($lotw_map as $cs => $lu) {
                $lotwdata[] = array('callsign' => $cs, 'lastupload' => $lu);
                if (count($lotwdata) >= $batch_size) {
                    if (!$this->db->insert_batch('lotw_users', $lotwdata)) {
                        echo "FAILED: Database error during batch insert";
                        log_message('error', 'Database error during LoTW batch insert while inserting deduped map');
                        return;
                    }
                    $batch_count++;
                    $lotwdata = array();
                }
            }
            // Final batch
            if (!empty($lotwdata)) {
                if (!$this->db->insert_batch('lotw_users', $lotwdata)) {
                    echo "FAILED: Database error during final batch insert";
                    log_message('error', 'Database error during LoTW final batch insert while inserting deduped map');
                    return;
                }
                $batch_count++;
            }
        }

        // Verify we actually imported data
        $final_count = $this->db->count_all('lotw_users');
        if ($final_count == 0) {
            echo "FAILED: No records were imported - possible data corruption";
            log_message('error', 'LoTW import completed but no records in database');
            return;
        }

        if ($i < 50000) { // Sanity check - LoTW should have at least 50k users
            echo "WARNING: Only imported " . $i . " records - this seems unusually low";
            log_message('warning', 'LoTW import record count unusually low: ' . $i);
        } 

        $mtime = microtime(); 
        $mtime = explode(" ",$mtime); 
        $mtime = $mtime[1] + $mtime[0]; 
        $endtime = $mtime; 
        $totaltime = ($endtime - $starttime); 
        echo "This page was created in ".$totaltime." seconds <br />"; 
        echo "Records read: " . $i . " <br/>";
        echo "Unique callsigns inserted: " . $final_count . " <br/>";
    }

    public function lotw_check() {
        $f = fopen('./updates/lotw_users.csv', "r");
        $result = false;
        while ($row = fgetcsv($f)) {
            if ($row[0] == '2M0SQL/MM') {
                $result = $row[0];
                echo "found";
                break;
            }
        }
        fclose($f);
    }

    /*
     * Used for autoupdating the DOK file which is used in the QSO entry dialog for autocompletion.
     */
    public function update_dok() {
        $contents = file_get_contents('https://www.df2et.de/cqrlog/dok_and_sdok.txt', true);

        if($contents === FALSE) {
            echo "Something went wrong with fetching the DOK file.";
        } else {
            $file = './assets/json/dok.txt';

            if (file_put_contents($file, $contents) !== FALSE) {     // Save our content to the file.
                $nCount = count(file($file));
                if ($nCount > 0)
                {
                    echo "DONE: " . number_format($nCount) . " DOKs and SDOKs saved";
                } else {
                    echo"FAILED: Empty file";
                }
            } else {
                echo"FAILED: Could not write to dok.txt file";
            }
        }
    }

    /*
     * Used for autoupdating the SOTA file which is used in the QSO entry dialog for autocompletion.
     */
    public function update_sota() {
        $csvfile = 'https://www.sotadata.org.uk/summitslist.csv';

        $sotafile = './assets/json/sota.txt';

        $csvhandle = fopen($csvfile,"r");
        if ($csvhandle === FALSE) {
            echo "Something went wrong with fetching the SOTA file";
            return;
        }

        $data = fgetcsv($csvhandle,1000,","); // Skip line we are not interested in
        $data = fgetcsv($csvhandle,1000,","); // Skip line we are not interested in
        $data = fgetcsv($csvhandle,1000,",");
        $sotafilehandle = fopen($sotafile, 'w');

        if ($sotafilehandle === FALSE) {
            echo"FAILED: Could not write to sota.txt file";
            return;
        }

        $nCount = 0;
        do {
            if ($data[0]) {
                fwrite($sotafilehandle, $data[0].PHP_EOL);
                $nCount++;
            }
        } while ($data = fgetcsv($csvhandle,1000,","));

        fclose($csvhandle);
        fclose($sotafilehandle);

        if ($nCount > 0)
        {
            echo "DONE: " . number_format($nCount) . " SOTA's saved";
        } else {
            echo"FAILED: Empty file";
        }
    }

    /*
     * Pulls the WWFF directory for autocompletion in QSO dialogs
     */
    public function update_wwff() {
        $csvfile = 'https://wwff.co/wwff-data/wwff_directory.csv';

        $wwfffile = './assets/json/wwff.txt';

        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $csvfile);
        curl_setopt($ch, CURLOPT_HEADER, false);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Cloudlog Updater');
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        $csv = curl_exec($ch);
        curl_close($ch);
        if ($csv === FALSE) {
            echo "Something went wrong with fetching the WWFF file";
            return;
        }

        $wwfffilehandle = fopen($wwfffile, 'w');
        if ($wwfffilehandle === FALSE) {
            echo"FAILED: Could not write to wwff.txt file";
            return;
        }

        $data = str_getcsv($csv,"\n");
        $nCount = 0;
        foreach ($data as $idx => $row) {
           if ($idx == 0) continue; // Skip line we are not interested in
           $row = str_getcsv($row, ',');
           if ($row[0]) {
              fwrite($wwfffilehandle, $row[0].PHP_EOL);
              $nCount++;
           }
        }

        fclose($wwfffilehandle);

        if ($nCount > 0)
        {
            echo "DONE: " . number_format($nCount) . " WWFF's saved";
        } else {
            echo"FAILED: Empty file";
        }
    }

    public function update_pota() {
        $csvfile = 'https://pota.app/all_parks.csv';

        $potafile = './assets/json/pota.txt';

        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $csvfile);
        curl_setopt($ch, CURLOPT_HEADER, false);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Cloudlog Updater');
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        $csv = curl_exec($ch);
        curl_close($ch);
        if ($csv === FALSE) {
            echo "Something went wrong with fetching the POTA file";
            return;
        }

        $potafilehandle = fopen($potafile, 'w');
        if ($potafilehandle === FALSE) {
            echo"FAILED: Could not write to pota.txt file";
            return;
        }
        $data = str_getcsv($csv,"\n");
        $nCount = 0;
        foreach ($data as $idx => $row) {
           if ($idx == 0) continue; // Skip line we are not interested in
           $row = str_getcsv($row, ',');
           if ($row[0]) {
              fwrite($potafilehandle, $row[0].PHP_EOL);
              $nCount++;
           }
        }

        fclose($potafilehandle);

        if ($nCount > 0)
        {
            echo "DONE: " . number_format($nCount) . " POTA's saved";
        } else {
            echo"FAILED: Empty file";
        }
    }

}
?>