#!/usr/bin/perl
## 
#  Usage   : IGS/MGEX data/products downloading
#  Created : Yuan Yongqiang, 2020/10/24
#  Updated : Yuan Yongqiang, 2020/10/25, adding EOP and PRODUCTS downloading
#
# -obs daily daily_v2 daily_v3 hourly hourly_v2 hourly_v3 
# -nav brdc hourly daily
# -dcb cas code (dlr not supported now)
# -snx igs
# -orb gfz wum grg cod igs
# -clk gfz wum grg cod igs
# -eop all daily gpsrapid
# -slr all
# 
## 1. Get a list of all files in a directory:
#  curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/*?list"
#
## 2. Get a list of files in a directory matching a pattern (cs2rx1700*.001.Z)
#  curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700*.001.Z?list"
#
## 3. Download a group of files within a range
#  curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700[1-9].001.Z" -O
#
## 4. Download a group of files from a list of specific files
#  curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700{1,3,7}.001.Z" -O
#
## 5. Download a set of files matching a pattern and output those to a single .tar file
#  curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx1700*.001.Z" -o files.tar
#
## 6. Download a single file
#  curl -c .urs_cookies -b .urs_cookies -n -L "https://cddis.nasa.gov/archive/doris/data/cs2/2017/cs2rx17001.001.Z" -O
#
 use strict;
 use warnings;
 use Net::FTP;
 use Getopt::Long;
 use File::Copy;
 use File::Path qw( make_path );
 use Data::Dumper;
 use File::Basename;
 use Class::Struct;
 use Cwd;
 use threads;
 use threads::shared;
 use Time::Local;
 
 struct DataType => { 
	 type => '$', ntype  => '$', 
	 time => '$', ctypes => '@',
 };
 struct IpMode => { 
	 ip   => '$', dir => '$',
	 mode => '$', type => '$',
	 subtype => '$',
 };
 struct CurlNode => { 
	 ip   => '$', user => '$', 
	 pwd  => '$', dir  => '$', 
	 file => '$', save => '$', 
 };
 
 my $year=0; 
 my $doy = 0;
 my $hour = 0;
 my $nthreads = 3;
 my $save = "";
 my @obs = ();
 my @nav = ();
 my @dcb = ();
 my @snx = ();
 my @orb = ();
 my @clk = ();
 my @eop = ();
 my @slr = ();
 GetOptions (
		"year=i" => \$year,
		"doy=i" => \$doy,
		"hour=i" => \$hour,
		"nth=i" => \$nthreads,
		"save=s" => \$save,
		"obs=s{,}" => \@obs,
		"nav=s{,}" => \@nav,
		"dcb=s{,}" => \@dcb,
		"snx=s{,}" => \@snx,
		"orb=s{,}" => \@orb,
		"clk=s{,}" => \@clk,
		"eop=s{,}" => \@eop,
		"slr=s{,}" => \@slr,
 );
 if($nthreads > 10) {
 	 print "Max threads 10\n";
 	 exit;
 }
 my @DataTypes = ();
 my $ctime = sprintf("%04d%03d%02d",$year,$doy,$hour);
 push @DataTypes,DataType->new(time=>$ctime, type=>"obs", ntype=>$#obs+1, ctypes=>\@obs) if(@obs);
 push @DataTypes,DataType->new(time=>$ctime, type=>"nav", ntype=>$#nav+1, ctypes=>\@nav) if(@nav);
 push @DataTypes,DataType->new(time=>$ctime, type=>"dcb", ntype=>$#dcb+1, ctypes=>\@dcb) if(@dcb);
 push @DataTypes,DataType->new(time=>$ctime, type=>"snx", ntype=>$#snx+1, ctypes=>\@snx) if(@snx);
 push @DataTypes,DataType->new(time=>$ctime, type=>"orb", ntype=>$#orb+1, ctypes=>\@orb) if(@orb);
 push @DataTypes,DataType->new(time=>$ctime, type=>"clk", ntype=>$#clk+1, ctypes=>\@clk) if(@clk);
 push @DataTypes,DataType->new(time=>$ctime, type=>"eop", ntype=>$#eop+1, ctypes=>\@eop) if(@eop);
 push @DataTypes,DataType->new(time=>$ctime, type=>"slr", ntype=>$#slr+1, ctypes=>\@slr) if(@slr);
 my @requested_CurlNodes = make_download_list(@DataTypes,$save);
 if(@requested_CurlNodes and ($#requested_CurlNodes+1) >= 1) {
 	 curl_download_files(@requested_CurlNodes,$nthreads);
 }
 exit;
 
 sub curl_download_files {
 	 my (@requested_CurlNodes) = @_;
 	 my $nthreads = pop(@requested_CurlNodes);
 	 
 	 my @cur_download_cmds = ();
 	 foreach my $requested_CurlNode (@requested_CurlNodes) {
 	 	 my $ip      = $requested_CurlNode->ip;
 	 	 my $user    = $requested_CurlNode->user;
 	 	 my $pwd     = $requested_CurlNode->pwd;
 	 	 my $rmtdir  = $requested_CurlNode->dir;
 	 	 my $file    = $requested_CurlNode->file;
 	 	 my $savedir = $requested_CurlNode->save;
 	 	 system("mkdir -p $savedir") if(! -d $savedir);
 	 	 my $cmd_curl = "";
 	 	 if($ip =~ /cddis/ or $ip =~ /http/) {
	 	   $cmd_curl = "curl -c .urs_cookies -b .urs_cookies -n --silent -L \"$ip$rmtdir/$file\" -o $savedir/$file";
	   }
	   else {
	   	 $cmd_curl = "curl $ip$rmtdir/$file -u anonymous:yqyuanwhu --silent -o $savedir/$file";
	   }
	   if(! -e "$savedir/$file") {
	   	 push @cur_download_cmds,$cmd_curl;
	   }
	   else {
	   	 print "Omitting already existed file: $savedir/$file\n";
	   }
	 }
	 
	 my @threads = ();
	 my $nfiles_per_thread = int(($#cur_download_cmds+1)/$nthreads);
	 $nthreads = 1 if($nfiles_per_thread<2);
	 #print "$nthreads\n";
	 #print "$nfiles_per_thread\n";
	 #exit;
	 for(my $ith = 0; $ith < $nthreads; $ith ++) {
     my $istart = $ith*$nfiles_per_thread;
     my $iend   = ($ith+1)*($nfiles_per_thread)-1;
     if($ith == $nthreads - 1) {
     	 $iend = $#cur_download_cmds;
     }
     #print "$istart  $iend\n";
	 	 my @th_cur_download_cmds = ();
	 	 for(my $icmd = $istart; $icmd <= $iend; $icmd ++) {
	 	 	 push @th_cur_download_cmds, $cur_download_cmds[$icmd];
	 	 }
	 	 if(@th_cur_download_cmds) {
	 	 	 my $th_cur = threads->create(\&run_download_files, @th_cur_download_cmds);
	 	 	 push @threads,$th_cur;
	 	 }
	 }
	 foreach my $curl_download_thread (@threads) {
	 	 $curl_download_thread->join();
	 }
 }
 
 sub run_download_files {
 	 my (@cur_download_cmds) = @_;
 	 foreach my $cur_download_cmd (@cur_download_cmds) {
 	 	 #print "$cur_download_cmd\n";
 	 	 my $time1=timelocal(localtime());
 	 	 system($cur_download_cmd);
 	 	 my $time2=timelocal(localtime());
 	 	 my $diff=$time2-$time1;
 	 	 print "Time consuming: $diff seconds; Finished: $cur_download_cmd\n";
 	 }
 }

 sub make_download_list {
 	 my (@DataTypes) = @_;
 	 my $save = pop(@DataTypes);
 	 my @IpModes = ();
 	 
 	 foreach my $DT (@DataTypes) {
	 	 my $year = substr($DT->time,0,4);
	 	 my $doy  = substr($DT->time,4,3);
	 	 my $hour = substr($DT->time,7,2);
	 	 my $type = $DT->type;
	 	 if($DT->ntype > 0) {
	 	 	 foreach my $subtype (@{$DT->ctypes}) {
	 	 	 	 push @IpModes,resolve_ip($year,$doy,$hour,$type,$subtype);
	 	 	 }
	 	 }
   }
   
   my @requested_CurlNodes = ();
   foreach my $ipnode (@IpModes) {
   	 my $ip = $ipnode->ip;
   	 my $dir = $ipnode->dir;
   	 my $mode = $ipnode->mode;
   	 my $type = $ipnode->type;
   	 my $subtype = $ipnode->subtype;
   	 print "Requesting files on remote server: ip = $ip, dir = $dir, mode = $mode, type = $type, subtype = $subtype\n";
   	 push @requested_CurlNodes, get_http_remote_list($year,$doy,$hour,$type,$subtype,$ip,$dir,$mode,$save);
   }
   return @requested_CurlNodes;
 }
 
 sub resolve_ip {
 	 my ($year,$doy,$hour,$type,$subtype) = @_;
 	 
 	 my $ip_cddis = "https://cddis.nasa.gov";
 	 my $ip_ign   = "ftp://igs.ign.fr";
 	 my $ip_whu   = "ftp://igs.gnsswhu.cn";
 	 my $ip_aiub  = "ftp://ftp.aiub.unibe.ch";
 	 my $ip_tum   = "ftp://edc.dgfi.tum.de";
 	 my $ip_iers  = "ftp://ftp.iers.org";
 	 
 	 my $mjd = yeardoy2mjd($year,$doy);
 	 my ($gwk,$gwkd) = mjd2gwkd($mjd);
 	 my ($year_not_used, $month, $monthday) = doy2monthday($year, $doy);
 	 
 	 my @IpModes = ();
# -obs cddis/ign/whu houly houly_v2 houly_v3 daily daily_v2 daily_v3
# -nav cddis/ign/whu brdc hourly daily
# -dcb cddis/aiub    cas dlr code
# -snx cddis/ign/whu igs
# -orb cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -clk cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -eop cddis/iers    all daily gpsrapid
# -slr cddis/tum     all prn
#### OBSERVATIONS
 	 if($type eq "obs") {
 	 	 my $mode_v2 = "*d.gz";
 	 	 my $mode_v3 = "*MO.crx.gz";
 	 	 my $dir_cddis = "";
 	 	 my $dir_ign   = "";
 	 	 my $dir_whu   = "";
 	 	 if($subtype =~ /hourly/) {
 	 	 	 $dir_cddis = sprintf("/archive/gnss/data/hourly/%04d/%03d/%02d",$year,$doy,$hour);
 	 	 	 $dir_ign   = sprintf("/pub/igs/data/hourly/%04d/%03d",$year,$doy);
 	 	 	 $dir_whu   = sprintf("/pub/gps/data/hourly/%04d/%03d/%02d",$year,$doy,$hour);
 	 	 }
 	 	 if($subtype =~ /daily/) {
 	 	 	 $dir_cddis = sprintf("/archive/gnss/data/daily/%04d/%03d/%02dd",$year,$doy,substr($year,2,2));
 	 	 	 $dir_ign   = sprintf("/pub/igs/data/%04d/%03d",$year,$doy);
 	 	 	 $dir_whu   = sprintf("/pub/gps/data/daily/%04d/%03d/%02dd",$year,$doy,substr($year,2,2));
 	 	 }
 	 	 if($subtype =~ /_v2/ or $subtype =~ /_v3/) {
 	 	 	 	 if($subtype =~ /_v2/) {
 	 	 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$mode_v2, type=>$type, subtype=>$subtype);
 	 	 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_ign,   dir=>$dir_ign,   mode=>$mode_v2, type=>$type, subtype=>$subtype);
 	 	 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_whu,   dir=>$dir_whu,   mode=>$mode_v2, type=>$type, subtype=>$subtype);
 	 	 	 	 }
 	 	 	 	 if($subtype =~ /_v3/) {
 	 	 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$mode_v3, type=>$type, subtype=>$subtype);
 	 	 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_ign,   dir=>$dir_ign,   mode=>$mode_v3, type=>$type, subtype=>$subtype);
 	 	 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_whu,   dir=>$dir_whu,   mode=>$mode_v3, type=>$type, subtype=>$subtype);
 	 	 	 	 }
 	 	 }
 	 	 else {
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$mode_v2, type=>$type, subtype=>$subtype);
 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_ign,   dir=>$dir_ign,   mode=>$mode_v2, type=>$type, subtype=>$subtype);
 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_whu,   dir=>$dir_whu,   mode=>$mode_v2, type=>$type, subtype=>$subtype);
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis, dir=>$dir_cddis, mode=>$mode_v3, type=>$type, subtype=>$subtype);
 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_ign,   dir=>$dir_ign,   mode=>$mode_v3, type=>$type, subtype=>$subtype);
 	 	 	 #push @IpModes,IpMode->new(ip=>$ip_whu,   dir=>$dir_whu,   mode=>$mode_v3, type=>$type, subtype=>$subtype);
 	 	 }
 	 }
#### NAVIGATION
 	 if($type eq "nav") {
 	 	 if($subtype =~ /brdc/) {
 	 	   my $dir = sprintf("/archive/gnss/data/daily/%04d/brdc",$year);
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>sprintf("*%02dn.gz",substr($year,2,2)), type=>$type, subtype=>$subtype); # GPS
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>sprintf("*%02dg.gz",substr($year,2,2)), type=>$type, subtype=>$subtype); # GLO
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>"BRD*MN.rnx.gz", type=>$type, subtype=>$subtype); # Multi-GNSS
 	 	 	 $dir = sprintf("/archive/gnss/data/daily/%04d/%03d/%02dp/",$year,$doy,substr($year,2,2));
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>"BRD*MN.rnx.gz", type=>$type, subtype=>$subtype); # Multi-GNSS
 	 	 }
 	 	 if($subtype =~ /hourly/) {
 	 	 	 my $dir = sprintf("/archive/gnss/data/hourly/%04d/%03d/%02d",$year,$doy,$hour);
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>"*N.rnx.gz", type=>$type, subtype=>$subtype); # Longname-only
 	 	 }
 	 	 if($subtype =~ /daily/) {
 	 	 	 my $dir = sprintf("/archive/gnss/data/daily/%04d/%03d/%02dp",$year,$doy,substr($year,2,2));
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>"*N.rnx.gz", type=>$type, subtype=>$subtype); # Longname-only
 	 	 }
 	 }
#### DCB
 	 if($type eq "dcb") {
 	 	 if($subtype =~ /cod/) {
 	 	 	 my $dir = sprintf("/CODE/%04d",$year);
 	 	 	 my $mode = "*.DCB.Z";
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_aiub,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 }
 	 	 if($subtype =~ /cas/) {
 	 	 	 my $dir = sprintf("/archive/gnss/products/bias/%04d",$year);
 	 	 	 my $mode = "*.BSX.gz";
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 }
 	 	 if($subtype =~ /dlr/) {
 	 	 	 # to be added
 	 	 }
 	 }
#### SINEX
 	 if($type eq "snx") {
 	 	 my $dir = sprintf("/archive/gnss/products/%04d",$gwk);
 	 	 my $mode = sprintf("igs*%04d.snx.Z",$gwk);
 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 $mode = sprintf("igs*%04d.snx.gz",$gwk);
 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 }
#### ORBIT/SP3
 	 if($type eq "orb") {
 	 	 if($subtype =~ /igs/) {
 	 	 	 my $dir = sprintf("/archive/gnss/products/%04d",$gwk);
 	 	 	 my $mode = "igs*.sp3.Z";
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 	 $mode = "igs*.sp3.gz";
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 }
 	 	 else {
 	 	 	 my $dir = sprintf("/archive/gnss/products/mgex/%04d",$gwk);
 	 	 	 my $mode = sprintf("%3s0MGXFIN*ORB.SP3.gz",uc($subtype));
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 	 $mode = sprintf("%3s0MGXRAP*ORB.SP3.gz",uc($subtype));
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 }
 	 }
#### CLOCK
 	 if($type eq "clk") {
 	 	 if($subtype =~ /igs/) {
 	 	 	 my $dir = sprintf("/archive/gnss/products/%04d",$gwk);
 	 	 	 my $mode = "igs*.clk_30s.Z";
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 	 $mode = "igs*.clk_30s.gz";
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 }
 	 	 else {
 	 	 	 my $dir = sprintf("/archive/gnss/products/mgex/%04d",$gwk);
 	 	 	 my $mode = sprintf("%3s0MGXFIN*CLK.CLK.gz",uc($subtype));
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 	 $mode = sprintf("%3s0MGXRAP*CLK.CLK.gz",uc($subtype));
 	 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 }
 	 }
#### EOP
 	 if($type eq "eop") {
 	 	 my $dir = "/archive/products/iers";
 	 	 my $mode = "";
 	 	 if($subtype =~ /daily/) {
 	 	 	 $mode = "*2000A.daily";
 	 	 	 #$dir = "/products/eop/rapid/daily";
 	 	 }
 	 	 if($subtype =~ /all/) {
 	 	 	 $mode = "*2000A.data";
 	 	 	 #$dir = "/products/eop/rapid/standard";
 	 	 }
 	 	 if($subtype =~ /gpsrapid/) {
 	 	 	 $mode = "gpsrapid*";
 	 	 	 #$dir = "/products/eop/rapid/standard";
 	 	 }
 	 	 push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	 #push @IpModes,IpMode->new(ip=>$ip_iers,    dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 }
#### SLR
 	 if($type eq "slr") {
 	 	 my @slr_sats = ("galileo101","galileo102","galileo103","galileo104","galileo201","galileo202","galileo203","galileo204",
                     "galileo205","galileo206","galileo207","galileo208","galileo209","galileo210","galileo211","galileo212",
                     "galileo213","galileo214","galileo215","galileo216","galileo217","galileo218","galileo219","galileo220",
                     "galileo221","galileo222","qzs1","qzs2","qzs3","qzs4","beidou3m2","beidou3m3","beidou3m9","beidou3m10",
                     "compassg1","compassi3","compassi5","compassi6b","compassm3");
     my $mode = sprintf("%04d%02d.npt",$year,$month);
 	 	 foreach my $slr_sat (@slr_sats) {
 	 	   my $dir = sprintf("/archive/slr/data/npt_crd/%s/%04d",$slr_sat,$year);
 	 	   push @IpModes,IpMode->new(ip=>$ip_cddis,   dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	 	   ## for TUM ftp
 	 	   #my $dir = sprintf("/pub/slr/data/npt_crd/%s/%04d",$slr_sat,$year);
 	 	   #push @IpModes,IpMode->new(ip=>$ip_tum,     dir=>$dir,   mode=>$mode, type=>$type, subtype=>$subtype);
 	   }
 	 }
 	 return @IpModes;
 }
 
 sub get_http_remote_list {
 	 my ($year,$doy,$hour,$type,$subtype,$ip,$dir,$mode,$save) = @_;
 	 my @requested_CurlNode = ();
 	 ## use http for cddis
 	 if($ip =~ /cddis/) {
 	 	 my @remote_lists = `curl -c .urs_cookies -b .urs_cookies -n -L --silent "$ip$dir/$mode?list"`;
 	 	 my $total        = `curl -c .urs_cookies -b .urs_cookies -n -L --silent "$ip$dir/$mode?list" | grep 'Total number of files'`;
 	 	 if(! $total) {
 	 	 	 print "No files!\n";
 	 	 	 return @requested_CurlNode;
 	 	 	 #exit;
 	 	 }
 	 	 else {
 	 	 	 foreach my $remote_list (@remote_lists) {
 	 	 	 	 my @token=split(/ /,$remote_list);
 	 	 	 	 my $filename = $token[0];
 	 	 	 	 if(looks_like_gnss($year,$doy,$hour,$type,$subtype,$filename)) {
 	 	 	 	 	 my $node = CurlNode->new(ip=>$ip,user=>"yqyuan",pwd=>"yqyuanwhu",dir=>$dir,file=>$filename,save=>$save);
 	 	 	 	 	 push @requested_CurlNode, $node; 
 	 	 	 	 }
 	 	 	 }
 	 	 }
 	 }
 	 ## use ftp for others
 	 else {
 	 	 my @remote_lists = `curl -u anonymous:yqyuanwhu --silent -l "$ip$dir/$mode"`;
 	 	 if(! @remote_lists) {
 	 	 	 print "No files!\n";
 	 	 	 return @requested_CurlNode;
 	 	 	 #exit;
 	 	 }
 	 	 else {
 	 	 	 foreach my $remote_list (@remote_lists) {
 	 	 	 	 my $filename = $remote_list;
 	 	 	 	 chomp($filename);
 	 	 	 	 if(looks_like_gnss($year,$doy,$hour,$type,$subtype,$filename)) {
 	 	 	 	 	 #print "$filename";
 	 	 	 	 	 my $node = CurlNode->new(ip=>$ip,user=>"yqyuan",pwd=>"yqyuanwhu",dir=>$dir,file=>$filename,save=>$save);
 	 	 	 	 	 push @requested_CurlNode, $node; 
 	 	 	 	 }
 	 	 	 }
 	 	 }
 	 }
 	 return @requested_CurlNode;
 }
 
 sub looks_like_gnss {
 	 my ($year,$doy,$hour,$type,$subtype,$remote_name) = @_;
 	 
 	 my $mjd = yeardoy2mjd($year,$doy);
 	 my ($gwk,$gwkd) = mjd2gwkd($mjd);
 	 my ($year_not_used, $month, $monthday) = doy2monthday($year, $doy);
 	 my @hour_id_basket = ("a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x");
	 my $hour_id = $hour_id_basket[$hour];
# -obs cddis/ign/whu houly houly_v2 houly_v3 daily daily_v2 daily_v3
# -nav cddis/ign/whu brdc hourly daily
# -dcb cddis/aiub    cas dlr code
# -snx cddis/ign/whu igs
# -orb cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -clk cddis/ign/whu gfz/gbm whu/wum grg cod igs
# -eop cddis/iers    all daily gpsrapid
# -slr cddis/tum     all prn
   my @ext_modes = ();
   if($type =~ /obs/) { 
   	 push @ext_modes,sprintf("%03d0.%02dd.Z",$doy,substr($year,2,2));
   	 push @ext_modes,sprintf("%03d%s.%02dd.Z",$doy,$hour_id,substr($year,2,2));
   	 push @ext_modes,sprintf("%03d0.%02dd.gz",$doy,substr($year,2,2));
   	 push @ext_modes,sprintf("%03d%s.%02dd.gz",$doy,$hour_id,substr($year,2,2));
   	 push @ext_modes,sprintf("%04d%03d0000_01D_30S_MO.crx.gz",$year,$doy); 
   	 push @ext_modes,sprintf("%04d%03d%02d00_01H_30S_MO.crx.gz",$year,$doy,$hour);
   }
   if($type =~ /nav/) { 
   	 #push @ext_modes,sprintf("%04d%03d%02d00_01H_GN.rnx.gz",$year,$doy,$hour);
   	 #push @ext_modes,sprintf("%04d%03d%02d00_01H_RN.rnx.gz",$year,$doy,$hour);
   	 #push @ext_modes,sprintf("%04d%03d%02d00_01H_EN.rnx.gz",$year,$doy,$hour);
   	 #push @ext_modes,sprintf("%04d%03d%02d00_01H_CN.rnx.gz",$year,$doy,$hour);
   	 #push @ext_modes,sprintf("%04d%03d%02d00_01H_JN.rnx.gz",$year,$doy,$hour);
   	 push @ext_modes,sprintf("%04d%03d%02d00_01H_MN.rnx.gz",$year,$doy,$hour);
   	 #push @ext_modes,sprintf("%03d%s.%02dn.Z",$doy,$hour_id,substr($year,2,2));
   	 #push @ext_modes,sprintf("%03d%s.%02dg.Z",$doy,$hour_id,substr($year,2,2));
   	 #push @ext_modes,sprintf("%03d%s.%02dl.Z",$doy,$hour_id,substr($year,2,2));
   	 #push @ext_modes,sprintf("%03d%s.%02df.Z",$doy,$hour_id,substr($year,2,2));
   	 #push @ext_modes,sprintf("%03d%s.%02dq.Z",$doy,$hour_id,substr($year,2,2));
   	 #push @ext_modes,sprintf("%03d%s.%02dp.Z",$doy,$hour_id,substr($year,2,2));
   	 push @ext_modes,sprintf("%04d%03d0000_01D_MN.rnx.gz",$year,$doy);
   	 push @ext_modes,sprintf("%03d0.%02dp.Z",$doy,substr($year,2,2));
   	 push @ext_modes,sprintf("%03d0.%02dp.gz",$doy,substr($year,2,2));
   }
   if($type =~ /dcb/) { 
   	 push @ext_modes,sprintf("%04d%03d0000_01D_01D_DCB.BSX.gz",$year,$doy);
   	 push @ext_modes,sprintf("P1C1%02d%02d.DCB.Z",substr($year,2,2),$month);
   	 push @ext_modes,sprintf("P1P2%02d%02d.DCB.Z",substr($year,2,2),$month);
   	 push @ext_modes,sprintf("P2C2%02d%02d_RINEX.DCB.Z",substr($year,2,2),$month);
   }
   if($type =~ /snx/) { 
   	 push @ext_modes,sprintf("%04d.snx.Z",$gwk);
   	 push @ext_modes,sprintf("%04d.snx.gz",$gwk);
   }
   if($type =~ /orb/) {
   	 push @ext_modes,sprintf("igs%04d%01d.sp3.Z",$gwk,$gwkd);
   	 push @ext_modes,sprintf("igs%04d%01d.sp3.gz",$gwk,$gwkd);
   	 push @ext_modes,sprintf("%04d%03d0000_01D_05M_ORB.SP3.gz",$year,$doy);
   	 push @ext_modes,sprintf("%04d%03d0000_01D_15M_ORB.SP3.gz",$year,$doy);
   }
   if($type =~ /clk/) { 
   	 push @ext_modes,sprintf("igs%04d%01d.clk_30s.Z",$gwk,$gwkd);
   	 push @ext_modes,sprintf("igs%04d%01d.clk_30s.gz",$gwk,$gwkd);
   	 push @ext_modes,sprintf("%04d%03d0000_01D_30S_CLK.CLK.gz",$year,$doy);
   }
   if($type =~ /eop/) { 
   	 push @ext_modes,"finals2000A.data";
   	 push @ext_modes,"finals2000A.daily";
   	 push @ext_modes,"gpsrapid.out";
   	 push @ext_modes,"gpsrapid.daily";
   }
   if($type =~ /slr/) { 
   	 push @ext_modes,sprintf("%04d%02d.npt",$year,$month);
   }
   my $lgnss = 0;
   foreach my $ext_mode (@ext_modes) {
	   if($remote_name =~ /$ext_mode/) {
	   	 $lgnss = 1;
	   	 last;
	   }
   }
   return $lgnss;
 }
 
 sub yeardoy2mjd {
	 my ($iyear, $idoy) = @_;
	 my $iyear2 = $iyear -1;
	 my $mjd;
	 {
		 use integer;
		 $mjd =  365*$iyear-678941+$iyear2/4-$iyear2/100+$iyear2/400;
	 } 
	 $mjd = $mjd + $idoy;
	 return $mjd;
 }
 
 sub mjd2gwkd {
	 my ($mjd) = @_;
	 my $gwk  = int(($mjd - 44244)/7);
	 my $gwkd = $mjd-44244 - $gwk * 7;
	 return ($gwk,$gwkd);
 }
 
 sub doy2monthday {
	 my @days_in_month = (31,28,31,30,31,30,31,31,30,31,30,31);
	 my ($iyear,$idoy) = @_;
	 $days_in_month[1]= ($iyear%4==0 && $iyear%100!=0 || $iyear%400 == 0) ? 29 : 28;
	 my $iday = $idoy;
	 my $imonth;
	 for($imonth=0;$imonth<12;$imonth++) {
		 if($iday <= $days_in_month[$imonth]) {
			 last;
		 }
		 $iday = $iday - $days_in_month[$imonth];
	 }
	 $imonth++;
	 return (sprintf("%04d",$iyear),sprintf("%02d",$imonth),sprintf("%02d",$iday));
 }