#!/usr/bin/perl
# This file is part of PWHOIS.
# #
# # The PWHOIS software provided in this Distribution is
# # Copyright 2005 VOSTROM Holdings, Inc.
# #
# # The full text of our legal notices is contained in the file called
# # COPYING, included with this Distribution
use strict;
use DBI;
use Getopt::Long;
use Time::Format;
use Log::Dispatch;
use Log::Dispatch::Screen;
use Log::Dispatch::File;
use Net::Telnet;

package Pwhoisd;

# globals
my $DEBUG = 'true';
my $VERSION = "1.1.2.25";
my $COPYRIGHT = 'Copyright (c) 2005-07 VOSTROM Holdings, Inc.';

# DO NOT MODIFY UNLESS YOU KNOW WHAT YOU ARE DOING: EDIT THE CONFIG FILE IF POSSIBLE
# See: /etc/pwhois/pwhoisd.conf or -c option to specify a different config file

my $DEFAULT_CONFIG = '/etc/pwhois/pwhoisd.conf';
my $DEFAULT_PIDFILE='/var/run/pwhoisd.pid';
my $DEFAULT_DATABASE_TYPE = 'postgres';
my $DEFAULT_DATABASE_SERVER = '';
my $DEFAULT_DATABASE_USER = 'pwhois';
my $DEFAULT_DATABASE_PASSWD = '';
my $DEFAULT_DATABASE_NAME = 'pwhois';

# connection settings for the remote download of BGP routes

my $DEFAULT_HOST='localhost';
my $DEFAULT_PORT=23;
my $DEFAULT_PASSWORD='pwhois';
my $DEFAULT_USER='pwhois';
my $DEFAULT_TIMEOUT=10800; # three hours
my $DEFAULT_RECORD_DB_CHUNK_SIZE=10000;
my $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE=100000;
my $DEFAULT_ROUTER_ID=0;   # used when reading from a file
my $MAX_ASNS=256;

# DO NOT EDIT BELOW THIS LINE

my $dbh;
my $dsn = 'NA';		# do not edit: these are changed later from defaults or config
my $user = 'NA';	# do not edit: these are changed later from defaults or config
my $password = 'NA';    # do not edit: these are changed later from defaults or config

my $select_from_bgp_routes_by_prefix_sth;
my $select_from_bgp_routes_by_prefix_nocidr_sth;
my $insert_into_bgp_routes_sth;
my $update_bgp_routes_sth;
my $update_bgp_routes_expired_sth;
my $select_from_asn_by_asn_sth;
my $insert_into_asn_sth;
my $update_asn_sth;
my $select_from_netblock_by_nethandle_sth;
my $insert_into_netblock_sth;
my $update_netblock_sth;
my $select_from_org_by_orgname_sth;
my $select_from_org_by_orgid_sth;
my $insert_into_org_sth;
my $update_org_sth;
my $select_from_poc_by_pochandle_sth;
my $insert_into_poc_sth;
my $update_poc_sth;

# define to skip over certain types of records within the ARIN data dump (used mostly for debugging purposes)
my $skipAsn = 0;
my $skipNet = 0;
my $skipOrg = 0;
my $skipPOC = 0;

# enviroment

$ENV{'TZ'} = 'GMT';

#log handle
my %config = ();
my $log;
my %opt = ();
#my %routes = ();

# install signal handlers
$SIG{HUP} = 'IGNORE';
$SIG{INT} = \&shutdown;
$SIG{TERM} = \&shutdown;
$SIG{KILL} = \&shutdown;
$SIG{QUIT} = 'IGNORE';    # ignore SIGQUIT
$SIG{__WARN__} =\&handle_warn;
#$SIG{__DIE__} = \&handle_error;

sub new {
   my $self = {};
   my $class = shift;
   return bless _init($self), $class
}

sub _init {
   my $self = shift;

   processArgs();
   
   openConfigFile();
  # connect to the database
   openDatabase();
  
   return $self
}

sub processArgs
{
        # change our process name in the PS list
        #$0 = "pwhoisd";

        Getopt::Long::Configure('no_ignore_case');
        Getopt::Long::GetOptions(\%opt, 'help|h', 'logfile|l=s', 'version|V', 'verbose|v+', 
				'configfile|c=s',
				'port|p=n', 'host=s', 'user|u=s', 'password|passwd|x=s', 'display-only', 
				'filter-by-source-as=n',
				'send-keep-alive',
				'infile|i=s',
				'router-id|r=s',
				'import-whois-dump'
                ) or die "Can't parse command-line options";
        usage() if $opt{help};

	# config file
	if(!defined($opt{'configfile'})) {
		$opt{'configfile'} = $DEFAULT_CONFIG;
	}

        # set version output to 0
        if(!defined($opt{verbose})) {
                $DEBUG = 'false';
		$opt{verbose} = 0;
        }
	else
	{
		if($opt{verbose} >= 2) {
                        $DEBUG = 'true';
                }
	}	

        if($opt{version}) {
                print "$0 $VERSION\n";
                print "Copyright (c) 2005 VOSTROM Holdings, Inc.\n";
                exit;
        }

	if($opt{logfile}) {
	        #  open our log file ... nothing should be written to stdout or stderr
        	# from now on

		if(! -f $opt{logfile}) {
			system("touch ". $opt{logfile});
		}        	
		$log = Log::Dispatch::File->new( name      => 'file1',
                                       min_level => 'debug',
                                       filename  => "$opt{logfile}",
                                       mode      => 'append');
                                       
        }
        else
        {
                  $log = Log::Dispatch::Screen->new( name      => 'screen',
                                                      min_level => 'debug',
                                                      stderr    => 0 );
        }

	# set the default port listenn on
	if(!defined($opt{'port'})) {
		$opt{'port'} = $DEFAULT_PORT;
	}
	
	# set the default host to connect to
	if(!defined($opt{'host'})) {
		$opt{'host'} = $DEFAULT_HOST;
	}
	
	# set the default host to connect to
	if(!defined($opt{'user'})) {
		$opt{'user'} = $DEFAULT_USER;
	}

	# set the default host to connect to
	if(!defined($opt{'password'})) {
		$opt{'password'} = $DEFAULT_PASSWORD;
	}

	# set the default router id (used only for reading from a file)
	if(!defined($opt{'router-id'})) {
		$opt{'router-id'} = $DEFAULT_ROUTER_ID;
	}

	$log->log( level => 'info', message => "$0 started at ". getDateTimeFormat(time()) ."\n") if $opt{verbose};
}


sub usage
{
        print "usage: $0 [*options*]\n\n",
              "  -h, --help         display this help and exit\n",
              "  -v, --verbose      be verbose about what you do\n",
              "  -V, --version      output version information and exit\n",
              "  -l, --logfile f    write misc progress output to logfile f instead of stdout\n",	     
      	      "  -c, --configfile f   read startup settings from configuration file: default is $DEFAULT_CONFIG\n",
	      "  --host <host>            The BGP host IP to connect to.\n",
	      "  --user|-u <user>         The user login name.\n",
	      "  --password|passwd <pass> The password for this user.\n",
	      "  --port <n>           The BGP host port to connect to: default: $DEFAULT_PORT\n",
	      "  --display-only		  Display the information on the screen only, don't put in db.\n",
	      "  --filter-by-source-as <asn>     this option when retreiving data from a local peer not a route-view\n",
	      "            and filter (select) only routes announced by this AS (useful so you can remove your AS from the path).\n",
	      "  -i, --infile f	  	  Read the BGP routes from the input file instead of connect over a socket\n",
	      "  -r, --router-id <id>	  When reading from a file (which doesn't have a router-id), useful to override\n",
	      "            the router-id used to updating the data in the database (default is 0)\n",
	      "  --send-keep-alive        Send keep-alive messages in session with route server\n",
		  "  --import-whois-dump      Read the WHOIS information (in ARIN dump format) from file (via -i) and import into database\n";
		
        exit;
}
# error or warning occurred, bail out
sub handle_warn($)
{
        my $msg = shift;
	if(defined($log)) {
	        $log->log(level=>'warning', message=>"Warning: $msg\n") if $opt{verbose} >= 1;
	}
	else
	{
	        print STDERR "Warning: $msg\n" if $opt{verbose} >= 1;

	}
}

sub handle_error($)
{
        my $msg = shift;
	if(defined($log)) {
	        $log->log(level=>'error', message=>"Error: $msg\n") if $opt{verbose} >= 1;
	}
	else
	{
	        print STDERR "Error: $msg\n" if $opt{verbose} >= 1;

	}
        &shutdown;
}

sub shutdown
{
	if(defined($log)) {
        	$log->log(level=>'info', message=>"$0 shutdown requested.\n") if $opt{verbose} >= 1;
       	} 
	closeDatabase();
        exit(0);
}

sub openConfigFile()
{
	if(defined($opt{'configfile'})) {
		if(-r $opt{'configfile'}) {
			open(CONFIG, $opt{'configfile'}) or die "Can't open config file: $!";
			while(<CONFIG>) {
				
				next if $_ =~ /^\s*#.*$/;  # skip comments
				next if $_ =~ /^\s+$/;	# skip blank lines
				
				my ($name,$value) = ($_ =~ /^\s*([A-z0-9_\-.]+) ?= ?"?([^"\r\n]*)"? ?$/); 
				my $display_value = $value;
				if($name =~ /^.*\.password$/i) {
					$display_value = '<hidden>';
				}
				$log->log(level=>"info", message=>"Found setting $name => $display_value\n") if $opt{verbose} >=1;
				
				$config{$name} = $value;				
			}
			close(CONFIG);
		}
	}
	else
	{
						
	}
	
	# configure based upon configuration settings
	
	my $database = $DEFAULT_DATABASE_TYPE;
	my $dbserver = $DEFAULT_DATABASE_SERVER;
	my $dbuser = $DEFAULT_DATABASE_USER;
	my $dbpasswd = $DEFAULT_DATABASE_PASSWD;
	my $dbname = $DEFAULT_DATABASE_NAME;
	
	if(defined($config{'db.type'})) {
		$database = $config{'db.type'};	
	}
	
	if(defined($config{'db.server'})) {
		$dbserver = $config{'db.server'};	
	}
	
	if(defined($config{'db.name'})) {
		$dbname = $config{'db.name'};	
	}
	
	if(defined($config{'db.user'})) {
		$dbuser = $config{'db.user'};
		$user = $dbuser;	
	}

	if(defined($config{'db.password'})) {
		$dbpasswd = $config{'db.password'};
		$password = $dbpasswd;
	}
	
	# configure the DSN
	
	if($database eq 'postgres' or 
	   $database eq 'pgsql' or 
	   $database eq 'postgresql')
	{
		if($dbserver ne '' and $dbserver ne 'localhost') {
				$dsn = "dbi:Pg:dbname=$dbname;host=$dbserver";
		}
		else {  # use the Unix Domain Socket (for localhost access)
			$dsn = "dbi:Pg:dbname=$dbname";
		}
	}
	elsif($database eq 'mysql')
	{
		$dsn = "dbi:mysql:dbname=$dbname:$dbserver";		
	}
	else
	{
		$log->log(level=>"error", message=>"Invalid database type specified '$database'\n");
		exit(0);
	}
	
		# configure other settings (config-file options override command-line except verbose)
	if(defined($config{'pwhois-updatedb.verbose'}) and !defined($opt{'verbose'}))
	{
		if($config{'pwhois-updatedb.verbose'} =~ /\d+/) {
			$opt{'verbose'} = $config{'pwhois-updatedb.verbose'};
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'pwhois-updatedb.verbose'\n");
			exit(-1);
		}
	}
	
	# log file on command-line overwrites 
	if(defined($config{'pwhois-updatedb.logfile'}) and !defined($opt{logfile}))
	{
		if($config{'pwhois-updatedb.logfile'} =~ /[A-z0-9\-_\/ \.]+/) {
			$opt{'logfile'} = $config{'pwhois-updatedb.logfile'};

			# close the current log
			# $log->close();

			# open the log file (it may have been open before on the console)
			
			if($opt{logfile}) {
			                #  open our log file ... nothing should be written to stdout or stderr
					#                  # from now on
					
					if(! -f $opt{logfile}) {
						system("touch ". $opt{logfile});
					}
					$log = Log::Dispatch::File->new( name      => 'file1',
									min_level => 'debug',
									filename  => "$opt{logfile}",
									mode      => 'append');
			}
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'pwhois-updatedb.logfile'\n");
			exit(-1);
		}
	}
	
	if(defined($config{'routeview.1'}))
	{
		if($config{'routeview.1'} =~ /[A-z0-9\-\.]+/) {
			$opt{'host'} = $config{'routeview.1'};
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'routeview.1'\n");
			exit(-1);
		}
	}
	
	if(defined($config{'routeview.1.port'}))
	{
		if($config{'routeview.1.port'} =~ /\d+/) {
			$opt{'port'} = $config{'routeview.1.port'};
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'routeview.1.port'\n");
			exit(-1);
		}
	}
	
	if(defined($config{'routeview.1.user'}))
	{
		if($config{'routeview.1.user'} =~ /\w+/) {
			$opt{'user'} = $config{'routeview.1.user'};
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'routeview.1.user'\n");
			exit(-1);
		}
	}
	
	if(defined($config{'routeview.1.password'}))
	{
		if($config{'routeview.1.password'} =~ /\w+/) {
			$opt{'password'} = $config{'routeview.1.password'};
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'routeview.1.password'\n");
			exit(-1);
		}
	}
	
	if(defined($config{'routeview.1.filter-by-source-as'}))
	{
		if($config{'routeview.1.filter-by-source-as'} =~ /\d+/) {
			$opt{'filter-by-source-as'} = $config{'routeview.1.filter-by-source-as'};
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'routeview.1.filter-by-source-as'\n");
			exit(-1);
		}
	}
	
	if(defined($config{'routeview.1.send-keep-alive'}))
	{
		if($config{'routeview.1.send-keep-alive'} == 0 or $config{'routeview.1.send-keep-alive'} == 1)
		{
			$opt{'send-keep-alive'} = $config{'routeview.1.send-keep-alive'};
		}
		else
		{
			$log->log(level=>"error", message=>"Invalid configuration option value for 'routeview.1.send-keep-alive'\n");
			exit(-1);
		}
	}	
}


# open the database and create prepared statements
sub openDatabase()
{	
	$dbh = DBI->connect($dsn, $user, $password,
                  { RaiseError => 1, AutoCommit => 0 }) 
		or die "Can't open database: $DBI::errstr";

	$log->log(level=>"debug", message=>"Database opened dsn=$dsn\n") if $DEBUG eq 'true'; 

	
	$select_from_bgp_routes_by_prefix_sth = $dbh->prepare(qq(
	  	SELECT id FROM bgp_routes 
		WHERE network=? AND cidr=? AND next_hop=? AND router_id=?
          	))
          	or die "Can't prepare statement: $DBI::errstr";
	
	$select_from_bgp_routes_by_prefix_nocidr_sth = $dbh->prepare(qq(
	 	SELECT id FROM bgp_routes 
		WHERE network=? AND cidr is null AND next_hop=? AND router_id=?
		))
		or die "Can't prepare statement: $DBI::errstr";

	$update_bgp_routes_expired_sth = $dbh->prepare(qq(
		UPDATE bgp_routes SET status=0 WHERE status=1 AND modifyDate < ? 
		)) or die "Can't prepare statement: $DBI::errstr";
 
	$insert_into_bgp_routes_sth = $dbh->prepare(qq(
      		INSERT INTO bgp_routes (router_id, network, cidr, next_hop, asn, asn_paths, createDate, modifyDate, status, best_route)
		VALUES (?,?,?,?,?,?,?,?,?,?)
      		))
		or die "Can't prepare statement: $DBI::errstr";

	$update_bgp_routes_sth = $dbh->prepare(qq(
		UPDATE bgp_routes SET asn=?, asn_paths=?, modifyDate=?, best_route=?, status=1
		WHERE id=?
		))
		or die "Can't prepare statement: $DBI::errstr";
		
	$select_from_asn_by_asn_sth = $dbh->prepare(qq(
	  	SELECT id FROM asn WHERE asn=?
          	))
          	or die "Can't prepare statement: $DBI::errstr";
	
	$insert_into_asn_sth = $dbh->prepare(qq(
      		INSERT INTO asn (asHandle, org_id, asn, asName, registerDate, comment, updateDate, techHandle, source, createDate, modifyDate)
			VALUES (?,?,?,?,?,?,?,?,?,?,?)
      		))
		or die "Can't prepare statement: $DBI::errstr";

	$update_asn_sth = $dbh->prepare(qq(
      		UPDATE asn SET asHandle=?, org_id=?, asn=?, asName=?, registerDate=?, comment=?, updateDate=?, 
			                techHandle=?, source=?, modifyDate=?
			WHERE id=?
      		))
		or die "Can't prepare statement: $DBI::errstr";
		
	$select_from_netblock_by_nethandle_sth = $dbh->prepare(qq(
	  	SELECT id FROM netblock WHERE netHandle=?
          	))
          	or die "Can't prepare statement: $DBI::errstr";
						
	$insert_into_netblock_sth = $dbh->prepare(qq(
      		INSERT INTO netblock (netHandle, org_id, parent, netName, netRange, network, netType, registerDate, comment, updateDate, 
							 nameserver1, nameserver2, nameserver3, nameserver4, nocHandle, abuseHandle, techHandle, 
							 source, createDate, modifyDate)
			VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
      		))
		or die "Can't prepare statement: $DBI::errstr";

	$update_netblock_sth = $dbh->prepare(qq(
      		UPDATE netblock SET netHandle=?, org_id=?, parent=?, netName=?, netRange=?, network=?, netType=?, registerDate=?, 
							          comment=?, updateDate=?, nameserver1=?, nameserver2=?, nameserver3=?, nameserver4=?, 
									  nocHandle=?, abuseHandle=?, techHandle=?, source=?, modifyDate=?
			WHERE id=?
      		))
		or die "Can't prepare statement: $DBI::errstr";

	$select_from_org_by_orgname_sth = $dbh->prepare(qq(
	  	SELECT id FROM organization WHERE orgName=?
          	))
          	or die "Can't prepare statement: $DBI::errstr";

	$select_from_org_by_orgid_sth = $dbh->prepare(qq(
	  	SELECT id FROM organization WHERE org_id=?
          	))
          	or die "Can't prepare statement: $DBI::errstr";
			
	$insert_into_org_sth = $dbh->prepare(qq(
      		INSERT INTO organization (org_id, orgName, canAllocate, street1, street2, street3, street4, street5, street6,
							 city, state, country, postalCode, registerDate, comment, updateDate, 
							 referralServer, adminHandle, nocHandle, abuseHandle, techHandle, source, createDate, modifyDate)
			VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
      		))
		or die "Can't prepare statement: $DBI::errstr";

	$update_org_sth = $dbh->prepare(qq(
      		UPDATE organization SET org_id=?, orgName=?, canAllocate=?, street1=?, street2=?, street3=?, street4=?, street5=?, street6=?, 
							    city=?, state=?, country=?, postalCode=?, registerDate=?, comment=?, updateDate=?, 
							    referralServer=?, adminHandle=?, nocHandle=?, abuseHandle=?, techHandle=?, source=?, modifyDate=?
			WHERE id=?
      		))
		or die "Can't prepare statement: $DBI::errstr";


	$select_from_poc_by_pochandle_sth = $dbh->prepare(qq(
	  	SELECT id FROM poc WHERE pocHandle=?
          	))
          	or die "Can't prepare statement: $DBI::errstr";


    $insert_into_poc_sth = $dbh->prepare(qq(
      		INSERT INTO poc (pocHandle, isRole, firstName, lastName, middleName, roleName, street1, street2, street3, street4, street5, street6,
							 city, state, country, postalCode, registerDate, comment, updateDate, officePhone, mailbox, source, 
							 createDate, modifyDate)
			VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
      		))
		or die "Can't prepare statement: $DBI::errstr";

	$update_poc_sth = $dbh->prepare(qq(
      		UPDATE poc SET pocHandle=?, isRole=?, firstName=?, lastName=?, middleName=?, roleName=?, street1=?, street2=?, street3=?, street4=?, street5=?, street6=?,
							 city=?, state=?, country=?, postalCode=?, registerDate=?, comment=?, updateDate=?, officePhone=?, mailbox=?, source=?, 
							modifyDate=?
			WHERE id=?
      		))
		or die "Can't prepare statement: $DBI::errstr";

}

# close down the database connection
sub closeDatabase()
{
	if(defined($log)) {
		$log->log(level=>"debug", message=>"Database closed dsn=$dsn\n") if $DEBUG eq 'true';
	}		
	undef $select_from_bgp_routes_by_prefix_sth;
	undef $insert_into_bgp_routes_sth;
	undef $update_bgp_routes_sth;
	undef $update_bgp_routes_expired_sth;
	undef $select_from_asn_by_asn_sth;
	undef $insert_into_asn_sth;
	undef $update_asn_sth;
	undef $select_from_netblock_by_nethandle_sth;
	undef $insert_into_netblock_sth;
	undef $update_netblock_sth;
	undef $select_from_org_by_orgname_sth;
	undef $select_from_org_by_orgid_sth;
	undef $insert_into_org_sth;
	undef $update_org_sth;
	undef $select_from_poc_by_pochandle_sth;
	undef $insert_into_poc_sth;
	undef $update_poc_sth;
	$dbh->disconnect if defined($dbh);

}

sub DESTROY {
   my $self = shift;
   # clean up lock and temp files
   #$self->SUPER::DESTROY
}

sub connect
{
        my $host = shift;
	my $login = shift;
	my $passwd = shift;


	my $session = Net::Telnet->new(timeout => 10, 
				       telnetmode=>0,
				       cmd_remove_mode => 1				       
				       );
	# change the buffer length
	$session->max_buffer_length(10048576);
				 
	$session->open(host=>$host, port=>$opt{'port'});	
	my @results = $session->print($passwd);
	print @results if $opt{verbose} >= 1;

	if($opt{verbose} >= 3) {
		if(!defined($opt{'logfile'})) {
			$session->dump_log();
		}
	}

	return $session;
}

sub disconnect
{
	my $session = shift;
	if(defined($session)) {
		$session->close();
	}
}

sub readBgpUpdate()
{
	# telnet to the BGP process running on the machine specified or read from a file
	my $lineno = 0;
	my $record = 0;
	my $router_id = $opt{'router-id'};  # default is 0 (if not defined by user) 
	my $cacheDate = time();			


	if(defined($opt{'infile'})) {
		# read from a file
		open(INPUT, $opt{'infile'}) or die "Can't read from input file: $!";
		
		$log->log(level=>'info', message=>"Reading from input file: ". $opt{'infile'} ."...\n") 
				if $opt{verbose} >= 1;
		
		while(<INPUT>)
		{
			($record, $lineno) = parseBgpUpdate(undef, $router_id, $lineno, $_, $record, $cacheDate);
			
			$log->log(level=>"info", message=>"$record records processed from $lineno lines\n") 
				if $opt{'verbose'} >= 1 and $lineno % $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE == 0;
			
			# commit the records to the database every 10000 records
        		if(!defined($opt{'display-only'}) and $lineno % $DEFAULT_RECORD_DB_CHUNK_SIZE == 0) {
				$dbh->commit() or warn "Can't commit records: $DBI::errstr";
			}
		}
	}
	else
	{ # read the data directly over the "telnet" connection to the shell

                if(defined($opt{'host'})) {
			if($opt{'host'} =~ /[A-z]+/) {
				my $ip =  resolve_hostname($opt{'host'});
				if($ip ne 'no address') {
					$router_id = $ip;
				}
				else
				{
					$log->log(level=>"error", message=>"Unable to resolve hostname of peer ". $opt{'host'} ."\n");
				}
			}
			else
			{
				# assume they gave us an IP
				$router_id = $opt{'host'};
			}
		}
		$log->log(level=>'info', message=>"connecting to ". $opt{host} .":". $opt{port} ." ...  router-id=$router_id\n") if $opt{verbose} >= 1;

		my $session;
		if(defined($opt{user})) { 
			$session = &connect($opt{host}, $opt{user}, $opt{password});
		}
		else
		{
			$session = &connect($opt{host});
		}
		
		return 0 if !defined($session);
		$log->log(level=>'info', message=>"Login successful -- retrieving data ...\n") if $opt{verbose} >= 1;

		my $buf;
		#$buf = $session->prompt('/^[#$>] ?$/');
		my @results = $session->cmd("terminal length 0");
		#$buf = $session->getline();
		#$buf = $session->getline();
		#$buf = $session->getline();
		$log->log(level=>"info", message=>"Sending command='show ip bgp' timeout=$DEFAULT_TIMEOUT errmode=return\n")
			if $opt{'verbose'} >= 1;

		$session->cmd(string=>"show ip bgp", timeout=>$DEFAULT_TIMEOUT, errmode=>"return");
		
		while(!$session->eof()) {
			$buf = $session->getline();
			if(!defined($buf)) {
				my $msg = $session->errmsg();
				$log->log(level=>"error", message=>"Error occurred during read: $msg\n");
				$session->close();
				last;
			}
			else
			{
				($record, $lineno) = parseBgpUpdate($session, $router_id, $lineno, $buf, $record, $cacheDate);
		
				$log->log(level=>"info", message=>"$record records processed from $lineno lines\n") 
					if $opt{'verbose'} >= 1 and $lineno % $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE == 0;
		
				if(defined($opt{'send-keep-alive'})) {
					if($lineno % $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE == 0)
					{
						$log->log(level=>'info', message=>"sending keep-alive ...\n") if $opt{verbose} >= 1;	
						$session->print("?");
					}
				}
			
				# commit the records to the database every 10000 records
        			if(!defined($opt{'display-only'}) and $lineno % $DEFAULT_RECORD_DB_CHUNK_SIZE == 0) {
					$dbh->commit() or warn "Can't commit records: $DBI::errstr";
				}
			
				if($buf =~ /^[A-z0-9.-]+> ?$/) 
				{
			 		$log->log(level=>'info', message=>"found command prompt.\n") if $opt{verbose} >= 2;	
					last if $record > 0;
				}		
			}
		}

		&disconnect();
	}
	
	my $elapsed_sec = time() - $cacheDate;
	
	$log->log(level=>'info', message=>"BGP routing table load completed in $elapsed_sec secs.  $lineno lines processed: $record routes updated.\n") if $opt{verbose};

	if(!defined($opt{'display-only'})) {
		# now update all the records which are no longer in the global routing table
		$update_bgp_routes_expired_sth->execute($cacheDate) 
			or warn "Can't update records: $DBI::errstr"; 
		$update_bgp_routes_expired_sth->finish();

		$dbh->commit() or warn "Can't commit records: $DBI::errstr";
	}
}

# used to store the last network found in the bgp routing table
my $last_network_found;

sub parseBgpUpdate($)
{
		my $session = shift;
		my $router_id = shift;
		my $line = shift;
		my $buf = shift;
		my $record = shift;
		my $createDate = shift;
		my $modifyDate = $createDate;

		if(defined($opt{'host'}) and $router_id eq '') {	
	
			if($opt{'host'} =~ /[A-z]+/) {
				my $ip =  resolve_hostname($opt{'host'});
				if($ip ne 'no address') {
					$router_id = $ip;
				}
				else
				{
					$log->log(level=>"error", message=>"Unable to resolve hostname of peer ". $opt{'host'} ."\n");
				}
			}
			else
			{
				# assume they gave us an IP
				$router_id = $opt{'host'};
			}		
		}

		# parser variables
		my $status = '';
		my $network = '';
		my $nextHop = '';
		my $metric = '';
		my $locPrf = '';
		my $weight = '';
		my $path = '';
		my $orgin = '';
		my $update = 1;
		my %routes = ();

		if(defined($opt{'display-only'})) {
			print STDOUT $buf;
		}

		$log->log(level=>'debug', message=>"processing line $line: $buf") if $DEBUG eq 'true' and $opt{verbose} >= 9;
		
			if ($buf =~ /^([sdh*i>]+)\s{1,5}([0-9\.\/]+)?\s+([0-9\.\/]+)\s+(\d+)?\s+(\d+)?\s+([0])\s+((\{?\,?\d+\s?\}?){1,$MAX_ASNS})\s+[ieshd?].*/) {
		    ($status, $network, $nextHop, $metric, $locPrf, $weight, $path)
			 = ($buf =~ /^([sdh*i>]+)\s{1,5}([0-9\.\/]+)?\s+([0-9\.\/]+)\s+(\d+)?\s+(\d+)?\s+([0])\s+((\{?\,?\d+\s?\}?){1,$MAX_ASNS})\s+[ieshd?].*/);

				# add the values to the hash

				# if there is no network/prefix listed on the log line (usually because there are more than one router for the same prefix)
				# get the last value and store that in the routing table -- at this point we need to mark which one is best
				if($network eq '') {
					$network = $last_network_found;
				}

				# skip routes that don't have a next-hop
				next if !defined($nextHop);

				$log->log(level=>'info', message=>"found network $network aspath $path \n")  if $opt{verbose} >= 5;

				my ($ipaddr,$cidr) = ipv4_parse($network);
				
				$routes{$network}{'class'} = 'bgp';
				if($status eq '*>') {
					$routes{$network}{'status'} = 1;
                                	$log->log(level=>'debug', message=>"found best route.\n") if $opt{'verbose'} >= 2;
				}
				else
				{
					$routes{$network}{'status'} = 0;
				}
				$routes{$network}{'route'} = $network;
				$routes{$network}{'network'} = $ipaddr;
				if($cidr ne '') {
					$routes{$network}{'network_cidr'} = $cidr;
				}

				$routes{$network}{'locpref'} = $locPrf;
				$routes{$network}{'nextHop'} = $nextHop;
				$routes{$network}{'metric'} = $metric;
				$routes{$network}{'weight'} = $weight;
				$path =~ s/{//g;
				$path =~ s/}//g;
				$path =~ s/,/ /g;
				# remove the first AS (this is ours)
				my(@asn_path) = split(/ /, $path);

				# if there are other routes in the routing table that we don't want to read
				# this option 
				#
				if(defined($opt{'filter-by-source-as'})) {
					next if $asn_path[0] != $opt{'filter-by-source-as'}; 
				}

				my @asn_path_most_specific = reverse(@asn_path);				
				$routes{$network}{'asn'} = $asn_path_most_specific[0];
				$routes{$network}{'source_asn'} = $asn_path[0];

				$last_network_found = $network;

				# if there is more than one AS listed, then remove our AS as it
				# is there only because we are getting the data from that provider
				# this also helps us hide our provider and not show where we are
				# getting our data from.
				if(scalar(@asn_path) > 1) {
					if(defined($opt{'filter-by-source-as'})) {
						shift(@asn_path);
					}
				}
				
				my $newpath = join ' ', @asn_path;
				$routes{$network}{'path'} = $newpath;      # most specific is on right side
  				$log->log(level=>'info', message=>"most specific ASN for $network is ". $routes{$network}{'asn'} 
							." next-hop=".  $routes{$network}{'nextHop'}  
							." best-route=". $routes{$network}{'status'} ."\n") if $opt{'verbose'} >= 2;

				$record++;	
			}  # under some cases (long ip addresses), output is wrapped.
			elsif ($buf =~ /^([sdh*i>]+)\s+([0-9.\/]+)$/) {
				$status = $1; $network = $2;
							
				my $newline;
				if(defined($session)) {	
					$newline = $session->getline(); 	# advance to the next line to read rest				
				}
				else
				{
					# read the next line from the input file
					$newline = <INPUT>;
				}				
				
				$line++;
				$log->log(level=>'debug', message=>"processing (next) line $line: $newline") if $DEBUG eq 'true' and $opt{verbose} >= 9;
				($nextHop, $metric, $locPrf, $weight, $path) =
					($newline =~ /^\s+([0-9\.\/]+)\s+(\d+)?\s+(\d+)?\s+([0])?\s+((\{?\,?\d+\s?\}?){1,$MAX_ASNS})\s+[ieshd?].*/);

				$log->log(level=>'info', message=>"found (wrapped) network $network aspath $path \n")  if $opt{verbose} >= 5;

				# add the values to the hash
				
				my ($ipaddr,$cidr) = ipv4_parse($network);
				
				$routes{$network}{'class'} = 'bgp';
				$routes{$network}{'route'} = $network;
				$routes{$network}{'network'} = $ipaddr;
				if($cidr ne '') {
					$routes{$network}{'network_cidr'} = $cidr;
				}
				
				$routes{$network}{'locpref'} = $locPrf;
				
				if($status eq '*>') {
					$routes{$network}{'status'} = 1;
				}
				else
				{
					$routes{$network}{'status'} = 0;
				}
				
				$routes{$network}{'nextHop'} = $nextHop;
				$routes{$network}{'metric'} = $metric;
				$routes{$network}{'weight'} = $weight;
				$path =~ s/{//g;
				$path =~ s/}//g;
				$path =~ s/,/ /g;
				# remove the first AS (this is ours)
				my(@asn_path) = split(/ /, $path);
				my @asn_path_most_specific = reverse(@asn_path);				
				$routes{$network}{'asn'} = $asn_path_most_specific[0];
				$routes{$network}{'source_asn'} = $asn_path[0];

				# if there is more than one AS listed, then remove our AS as it
				# is there only because we are getting the data from that provider
				# this also helps us hide our provider and not show where we are
				# getting our data from.
				if(scalar(@asn_path) > 1) {
					if(defined($opt{'filter-by-source-as'})) {
				        	shift(@asn_path);
					}
				}
				
				my $newpath = join ' ', @asn_path;
				$routes{$network}{'path'} = $newpath;      # most specific is on right side
			 	$log->log(level=>'info', message=>"most specific ASN for $network is ". $routes{$network}{'asn'}
			                                                         ." next-hop=".  $routes{$network}{'nextHop'}
										 ." best-route=". $routes{$network}{'status'} ."\n") 
										 if $opt{'verbose'} >= 2; 
				$record++;	
			}
			else
			{
				chomp($_);
				$log->log(level=>'debug', message=>"skipping line ... $buf \n") if $opt{verbose} >= 9;
				$update = 0;
			}
		
			# only put in the database if display-only flag isn't set
			if(!defined($opt{'display-only'})) {
				if($update) {
					my $sth;
					my $found_record=0;
					my $record_id=0;
					if(defined($routes{$network}{'network_cidr'})) {

						# determine the record already exists within the database
						$select_from_bgp_routes_by_prefix_sth->execute(
							ipv4_quaddot_to_decimal($routes{$network}{'network'}),
							$routes{$network}{'network_cidr'},
							ipv4_quaddot_to_decimal($routes{$network}{'nextHop'}),
							ipv4_quaddot_to_decimal($router_id)
							) or die "Can't select record: $DBI::errstr";
						$sth = $select_from_bgp_routes_by_prefix_sth;
						while(my($id) = $select_from_bgp_routes_by_prefix_sth->fetchrow_array()) {
							$found_record=1;
							$record_id = $id;
						}
						$select_from_bgp_routes_by_prefix_sth->finish();
					}
					else
					{
						# determine the record already exists within the database
						$select_from_bgp_routes_by_prefix_nocidr_sth->execute(
							ipv4_quaddot_to_decimal($routes{$network}{'network'}),
							ipv4_quaddot_to_decimal($routes{$network}{'nextHop'}),
							ipv4_quaddot_to_decimal($router_id)
							) or die "Can't select record: $DBI::errstr";
						while(my($id) = $select_from_bgp_routes_by_prefix_nocidr_sth->fetchrow_array()) {
							$found_record=1;
							$record_id = $id;
						}
						$select_from_bgp_routes_by_prefix_nocidr_sth->finish();
					}
			
					if($found_record) {
						$log->log(level=>'debug', message=>"Found record for update id=$record_id\n")
							if $opt{'verbose'} >= 5;

						# update the record: set the source-as, path, date of modification, and any
						# change in status == meaning this path may no longer be the best path
						$update_bgp_routes_sth->execute(
						 	$routes{$network}{'asn'},
						 	$routes{$network}{'path'},
						 	$modifyDate,
						 	$routes{$network}{'status'},  # best_route
							$record_id
						) or die "Can't update record: $DBI::errstr";
						$update_bgp_routes_sth->finish();
						$log->log(level=>'debug', message=>"updating record $record_id in bgp_routes for ". 
								$routes{$network}{'network'} ."/". $routes{$network}{'network_cidr'}
								." next-hop=". $routes{$network}{'nextHop'} 
								." with source-as=". $routes{$network}{'asn'}
								." as-path=". $routes{$network}{'path'}
								." best-route=". $routes{$network}{'status'} 
								."\n") if $opt{verbose} >= 4;
					}
					else
					{
						# insert the record into the database
						$insert_into_bgp_routes_sth->execute(
							ipv4_quaddot_to_decimal($router_id),
							ipv4_quaddot_to_decimal($routes{$network}{'network'}),
							$routes{$network}{'network_cidr'},
							ipv4_quaddot_to_decimal($routes{$network}{'nextHop'}),
							$routes{$network}{'asn'},
							$routes{$network}{'path'},
							$createDate,
							$modifyDate,
							1,		# status is '1' = "Active"
							$routes{$network}{'status'}
						) or die "Can't insert record: $DBI::errstr";
						$insert_into_bgp_routes_sth->finish();
						$log->log(level=>'debug', message=>"inserting record into bgp_routes with ". 
								$routes{$network}{'network'} ."/". $routes{$network}{'network_cidr'}
								." next-hop=". $routes{$network}{'nextHop'} 
								." source-as=". $routes{$network}{'asn'}
								." as-path=". $routes{$network}{'path'}
								." best-route=". $routes{$network}{'status'} 
								."\n") if $opt{verbose} >= 4;
					}
				}
			}
								
		return ($record, ++$line);
}

# read a ARIN formatted whois update dump file and import it into the database
sub readWhoisUpdate
{
	my $lineno = 0;
	my $record = 0;
	my $record_id;
	my $cacheDate = time();			

	if(defined($opt{'infile'}))
	{
		# read from a file
		open(INPUT, $opt{'infile'}) or die "Can't read from input file: $!";
		
		$log->log(level=>'info', message=>"Reading from input file: ". $opt{'infile'} ."...\n") 
				if $opt{verbose} >= 1;
	
		my %rec = ();
		while(<INPUT>)
		{			
			$lineno++;
		#	print $_ if $opt{verbose} >= 2;
			next if /^\s*#$/;
			next if /^\s*$/;
			
			my $found_asn = 0;
			my $found_net = 0;
			my $found_org = 0;
			my $found_poc = 0;
			
			my $ncomment=0;	
			my $nnameserver=1;
			my $nstreet=1;
			
			if($_ =~ /^\s*ASHandle:/)
			{
				$found_asn = 1;
				# parse asn information
				%rec = ();
				my $nextline = $_;
		#		print "found as handle, parsing.  $nextline";
				while($nextline =~ /^\s*[A-z\\\/]+\:.*$/) {
					if($nextline =~ /^\s*ASHandle:\s+([A-z0-9\-\.]+)$/) {
						$rec{'asHandle'} = $1;
					}
					elsif($nextline =~ /^\s*ASNumber:\s*(\d+)$/) {
						$rec{'asn'} = $1;
					}
					# two different formats, one with just a number, this one with a range of numbers (only storing the first one)
					elsif($nextline =~ /^\s*ASNumber:\s*(\d+) - \d+$/) {
						$rec{'asn'} = $1;
					}
					elsif($nextline =~ /^\s*OrgID:\s*(.*)$/) {
						$rec{'orgId'} = $1;
					}
					elsif($nextline =~ /^\s*ASName:\s*(.*)$/) {
						$rec{'asName'} = $1
					}
					elsif($nextline =~ /^\s*RegDate:\s*(.*)$/) {
						if($1 ne '') {
							$rec{'registerDate'} = $1;
						}
						else {
							$rec{'registerDate'} = '1970-01-01';						
						}
					}
					elsif($nextline =~ /^\s*Updated:\s*(.*)$/) {
						$rec{'updateDate'} = $1;
					}
					elsif($nextline =~ /^\s*Comment:\s*(.*)$/) {
						$rec{'comment'} = $1 if $ncomment == 0;
						$rec{'comment'} .= " ". $1 if $ncomment > 0;
						$ncomment++;
					}
					elsif($nextline =~ /^\s*Source:\s*(.*)$/) {
						$rec{'source_string'} = $1;
						if($1 eq 'ARIN') {
							$rec{'source'} = 1;
						}
						elsif($1 eq 'RIPE') {
							$rec{'source'} = 2;
						}
						elsif($1 eq 'APNIC') {
							$rec{'source'} = 3;
						}
						else { # unknown
							$rec{'source'} = 0;
						}
					}
					elsif($nextline =~ /^\s*TechHandle:\s*(.*)$/) {
						$rec{'techHandle'} = $1;
					}
				
				# advance to the next line
				$nextline = <INPUT>;
				$lineno++;
	#			print "reading next line: $nextline";
				}
				
				$record++;
				
				$log->log(level=>'error', message=>"ERROR: Record data not parsed properly for last asn\n") if !defined($rec{'asn'});
				
				my $found_record=0;
				if(defined($rec{'asn'}) and !$skipAsn) {
					# update/insert into database
					 $select_from_asn_by_asn_sth->execute($rec{'asn'}) or die "Can't select record: $DBI::errstr";;
					 while(my($id) = $select_from_asn_by_asn_sth->fetchrow_array()) {
						$found_record=1;
						$record_id = $id;
					 }
					 
					 $select_from_asn_by_asn_sth->finish();
					 
					 if($found_record) {
						
						$log->log(level=>'debug', message=>"Found AS record: ". $rec{'asn'} ." in database -- updating ... id=". $record_id ."\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;
						$update_asn_sth->execute($rec{'asHandle'}, $rec{'orgId'}, $rec{'asn'}, 
												      $rec{'asName'}, $rec{'registerDate'}, $rec{'comment'},
													  $rec{'updateDate'}, $rec{'techHandle'}, $rec{'source'},
													  $cacheDate, $record_id)
								or die "Can't select record: $DBI::errstr";
						$update_asn_sth->finish();
					 }
					 else {
						$log->log(level=>'debug', message=>"Found AS record: ". $rec{'asn'} ." but not in database -- inserting ...". $record_id ."\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;

						$insert_into_asn_sth->execute($rec{'asHandle'}, $rec{'orgId'}, $rec{'asn'}, 
												      $rec{'asName'}, $rec{'registerDate'}, $rec{'comment'},
													  $rec{'updateDate'}, $rec{'techHandle'}, $rec{'source'},
													  $cacheDate, $cacheDate)
								or die "Can't select record: $DBI::errstr";
						
						$insert_into_asn_sth->finish();
					 }
				}
				
				$log->log(level=>"info", message=>"$record AS records processed from $lineno lines\n") if $opt{'verbose'} >= 1 and $lineno % $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE == 0;
			
				# commit the records to the database every N records
				if(!defined($opt{'display-only'}) and $record % $DEFAULT_RECORD_DB_CHUNK_SIZE == 0) {
					$dbh->commit() or warn "Can't commit records: $DBI::errstr";
				}
			} # end if parser
			elsif($_ =~ /^\s*NetHandle:/)
			{
				$found_net = 1;
			
				# parse netblock information
				%rec = ();
				
				# set defaults
				$rec{'netType'} = 0;
				$rec{'netName'} = 'UNDEFINED';
				
				my $nextline = $_;
	#			print "found net handle, parsing.  $nextline";
				while($nextline =~ /^\s*[A-z\\\/]+\:.*$/) {
					if($nextline =~ /^\s*NetHandle:\s+([A-z0-9\-]+)$/) {
						$rec{'netHandle'} = $1;
					}
					elsif($nextline =~ /^\s*OrgID:\s*(.*)$/) {
						$rec{'orgId'} = $1;
					}
					elsif($nextline =~ /^\s*Parent:\s*(.*)$/) {
						$rec{'parent'} = $1;
					}
					elsif($nextline =~ /^\s*NetName:\s*(.*)$/) {
						$rec{'netName'} = $1;
					}
					elsif($nextline =~ /^\s*NetRange:\s*(.*)$/) {
						$rec{'netRange'} = $1;
						
						my ($network) = ($1 =~ /^(\d+\.\d+\.\d+\.\d+) - .*$/);
				#		print "found network $network from ". $rec{'netRange'} ."\n";
						if(defined($network)) {
							$rec{'network'} = $network;
						}
					}
					elsif($nextline =~ /^\s*NetType:\s*(.*)$/) {
						if($1 eq 'assignment') {
							$rec{'netType'} = 1;
						}
						elsif($1 eq 'reassignment') {
							$rec{'netType'} = 2;
						}
						elsif($1 eq 'rir') {
							$rec{'netType'} = 3;
						}
						elsif($1 eq 'allocation') {
							$rec{'netType'} = 4;
						}
						else {
							$rec{'netType'} = 0;
						}
					}
					elsif($nextline =~ /^\s*RegDate:\s*(.*)$/) {
						if($1 ne '') {
							$rec{'registerDate'} = $1;
						}
						else {
							$rec{'registerDate'} = '1970-01-01';						
						}
					}
					elsif($nextline =~ /^\s*Updated:\s*(.*)$/) {
						$rec{'updateDate'} = $1;
					}
					elsif($nextline =~ /^\s*NameServer:\s*(.*)$/) {
						$rec{"nameserver$nnameserver"} = $1;
						$nnameserver++;
					}
					elsif($nextline =~ /^\s*Comment:\s*(.*)$/) {
						$rec{'comment'} = $1 if $ncomment == 0;
						$rec{'comment'} .= " ". $1 if $ncomment > 0;
						$ncomment++;
					}
					elsif($nextline =~ /^\s*Source:\s*(.*)$/) {
						$rec{'source_string'} = $1;
						if($1 eq 'ARIN') {
							$rec{'source'} = 1;
						}
						elsif($1 eq 'RIPE') {
							$rec{'source'} = 2;
						}
						elsif($1 eq 'APNIC') {
							$rec{'source'} = 3;
						}
						else { # unknown
							$rec{'source'} = 0;
						}
					}
					elsif($nextline =~ /^\s*TechHandle:\s*(.*)$/) {
						$rec{'techHandle'} = $1;
					}
					
					# advance to the next line
					$nextline = <INPUT>;
					$lineno++;
		#			print "reading next line: $nextline";
				}
				
				$record++;
				
				$log->log(level=>'error', message=>"ERROR: Record data not parsed properly for last nethandle\n") if !defined($rec{'netHandle'});
								
				my $found_record=0;
				if(defined($rec{'netHandle'}) and !$skipNet) {
					# update/insert into database
					 $select_from_netblock_by_nethandle_sth->execute($rec{'netHandle'}) or die "Can't select record: $DBI::errstr";;
					 while(my($id) = $select_from_netblock_by_nethandle_sth->fetchrow_array()) {
						$found_record=1;
						$record_id = $id;
					 }
					 
					 $select_from_netblock_by_nethandle_sth->finish();
					 
					 if($found_record) {
						
						$log->log(level=>'debug', message=>"Found NetHandle record: ". $rec{'netHandle'} ." in database -- updating id=". $record_id ."\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;
						$update_netblock_sth->execute($rec{'netHandle'}, $rec{'orgId'}, $rec{'parent'}, 
												      $rec{'netName'}, $rec{'netRange'}, ipv4_quaddot_to_decimal($rec{'network'}), $rec{'netType'}, 
													  $rec{'registerDate'}, $rec{'comment'}, $rec{'updateDate'}, 
													  $rec{'nameserver1'}, $rec{'nameserver2'}, $rec{'nameserver3'}, $rec{'nameserver4'},
													  $rec{'nocHandle'}, $rec{'abuseHandle'}, $rec{'techHandle'}, $rec{'source'}, $cacheDate, 
													  $record_id)
								or die "Can't select record: $DBI::errstr";
						$update_netblock_sth->finish();
					 }
					 else {
						$log->log(level=>'debug', message=>"Found NetHandle record: ". $rec{'netHandle'} ." but not in database -- inserting ...\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;
						$insert_into_netblock_sth->execute($rec{'netHandle'}, $rec{'orgId'}, $rec{'parent'}, 
												           $rec{'netName'}, $rec{'netRange'}, ipv4_quaddot_to_decimal($rec{'network'}), $rec{'netType'}, 
													       $rec{'registerDate'}, $rec{'comment'}, $rec{'updateDate'}, 
													       $rec{'nameserver1'}, $rec{'nameserver2'}, $rec{'nameserver3'}, $rec{'nameserver4'},
													       $rec{'nocHandle'}, $rec{'abuseHandle'}, $rec{'techHandle'}, $rec{'source'},
														   $cacheDate, $cacheDate)
								or die "Can't select record: $DBI::errstr";
						
						$insert_into_netblock_sth->finish();
					 }
				}
			
				$log->log(level=>"info", message=>"$record NetHandle records processed from $lineno lines\n") if $opt{'verbose'} >= 1 and $lineno % $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE == 0;

				# commit the records to the database every N records
				if(!defined($opt{'display-only'}) and $record % $DEFAULT_RECORD_DB_CHUNK_SIZE == 0) {
					$dbh->commit() or warn "Can't commit records: $DBI::errstr";
				}
			}
			elsif($_ =~ /^\s*OrgID:/ and !$found_asn and !$found_net)			# make sure we didn't find the OrgID inside a ASN or Net record
			{
				$found_org = 1;
				# parse org information
				%rec = ();
				my $nextline = $_;
				#print "found org handle, parsing.  $nextline";
				while($nextline =~ /^\s*[A-z\\\/]+\:.*$/) {
					
				#	print "found parsing nextline:  $nextline";

					if($nextline =~ /^\s*OrgID:\s+(.*)$/) {
						$rec{'orgId'} = $1;
					}
					elsif($nextline =~ /^\s*OrgName:\s+(.*)$/) {
						$rec{'orgName'} = $1;
					}
					elsif($nextline =~ /^\s*CanAllocate:\s*(.*)$/) {
						if($1 eq 'Y') {
							$rec{'canAllocate'} = 1;
						}
						else {
							$rec{'canAllocate'} = 0;
						}
					}
					elsif($nextline =~ /^\s*Street:\s*(.*)$/) {
						$rec{"street$nstreet"} = $1;
						$nstreet++;
					}
					elsif($nextline =~ /^\s*City:\s*(.*)$/) {
						$rec{'city'} = $1
					}
					elsif($nextline =~ /^\s*State\/Prov:\s*(.*)$/) {
						$rec{'state'} = $1;
					}
					elsif($nextline =~ /^\s*Country:\s*(.*)$/) {
						$rec{'country'} = $1;
					}
					elsif($nextline =~ /^\s*PostalCode:\s*(.*)$/) {
						$rec{'postalCode'} = $1;
					}
					elsif($nextline =~ /^\s*RegDate:\s*(.*)$/) {
						if($1 ne '') {
							$rec{'registerDate'} = $1;
						}
						else {
							$rec{'registerDate'} = '1970-01-01';						
						}
					}
					elsif($nextline =~ /^\s*Updated:\s*(.*)$/) {
						$rec{'updateDate'} = $1;
					}
					elsif($nextline =~ /^\s*ReferralServer:\s*(.*)$/) {
						$rec{'referralServer'} = $1;
					}
					elsif($nextline =~ /^\s*Comment:\s*(.*)$/) {
						$rec{'comment'} = $1 if $ncomment == 0;
						$rec{'comment'} .= " ". $1 if $ncomment > 0;
						$ncomment++;
					}
					elsif($nextline =~ /^\s*Source:\s*(.*)$/) {
						$rec{'source_string'} = $1;
						if($1 eq 'ARIN') {
							$rec{'source'} = 1;
						}
						elsif($1 eq 'RIPE') {
							$rec{'source'} = 2;
						}
						elsif($1 eq 'APNIC') {
							$rec{'source'} = 3;
						}
						else { # unknown
							$rec{'source'} = 0;
						}
					}
					elsif($nextline =~ /^\s*OrgAbuseHandle:\s*(.*)$/) {
						$rec{'abuseHandle'} = $1;
					}
					elsif($nextline =~ /^\s*OrgAdminHandle:\s*(.*)$/) {
						$rec{'adminHandle'} = $1;
					}
					elsif($nextline =~ /^\s*OrgNOCHandle:\s*(.*)$/) {
						$rec{'nocHandle'} = $1;
					}
					elsif($nextline =~ /^\s*OrgTechHandle:\s*(.*)$/) {
						$rec{'techHandle'} = $1;
					}
		
					# advance to the next line
					$nextline = <INPUT>;
					$lineno++;
					#print "reading next line: $nextline";
				}
				
				$record++;
												
				my $found_record=0;
				if(defined($rec{'orgName'}) and defined($rec{'orgId'}) and !$skipOrg) {
					# update/insert into database
					 $select_from_org_by_orgid_sth->execute($rec{'orgId'}) or die "Can't select record: $DBI::errstr";;
					 while(my($id) = $select_from_org_by_orgid_sth->fetchrow_array()) {
						$found_record=1;
						$record_id = $id;
					 }
					 
					 $select_from_org_by_orgid_sth->finish();
					 
					 if($found_record) {
						
						$log->log(level=>'debug', message=>"Found Org record: ". $rec{'orgId'} ." in database -- updating id=". $record_id ."\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;
						$update_org_sth->execute($rec{'orgId'}, $rec{'orgName'}, $rec{'canAllocate'}, 
												      $rec{'street1'}, $rec{'street2'}, $rec{'street3'}, $rec{'street4'}, $rec{'street5'}, $rec{'street6'},
													  $rec{'city'}, $rec{'state'}, $rec{'country'}, $rec{'postalCode'}, $rec{'registerDate'}, $rec{'comment'}, $rec{'updateDate'}, 
													  $rec{'referralServer'}, $rec{'adminHandle'}, $rec{'nocHandle'}, $rec{'abuseHandle'}, $rec{'techHandle'}, 
													  $rec{'source'}, $cacheDate, 
													  $record_id)
								or die "Can't select record: $DBI::errstr";
						$update_org_sth->finish();
					 }
					 else {
						$log->log(level=>'debug', message=>"Found Org record: ". $rec{'orgId'} ." but not in database -- inserting ...\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;
						$insert_into_org_sth->execute($rec{'orgId'}, $rec{'orgName'}, $rec{'canAllocate'}, 
												      $rec{'street1'}, $rec{'street2'}, $rec{'street3'}, $rec{'street4'}, $rec{'street5'}, $rec{'street6'},
													  $rec{'city'}, $rec{'state'}, $rec{'country'}, $rec{'postalCode'}, $rec{'registerDate'}, $rec{'comment'}, $rec{'updateDate'}, 
													  $rec{'referralServer'}, $rec{'adminHandle'}, $rec{'nocHandle'}, $rec{'abuseHandle'}, $rec{'techHandle'}, 
													  $rec{'source'}, $cacheDate, $cacheDate) or die "Can't select record: $DBI::errstr";
						
						$insert_into_org_sth->finish();
					 }
				}
				else {
					$log->log(level=>'error', message=>"ERROR: Record data not parsed properly for last org\n");
				}
			
				$log->log(level=>"info", message=>"$record Org records processed from $lineno lines\n") if $opt{'verbose'} >= 1 and $lineno % $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE == 0;

				# commit the records to the database every N records
				if(!defined($opt{'display-only'}) and $record % $DEFAULT_RECORD_DB_CHUNK_SIZE == 0) {
					$dbh->commit() or warn "Can't commit records: $DBI::errstr";
				}
			}
			elsif($_ =~ /^\s*POCHandle:/)
			{
				$found_poc = 1;
				# parse org information
				%rec = ();
				my $nextline = $_;
				print "found POC handle, parsing.  $nextline";
				while($nextline =~ /^\s*[A-z\\\/]+\:.*$/) {
					if($nextline =~ /^\s*POCHandle:\s+([A-z0-9\-]+)$/) {
						$rec{'pocHandle'} = $1;
					}
					elsif($nextline =~ /^\s*IsRole:\s+(.*)$/) {
						if($1 eq 'Y') {
							$rec{'isRole'} = 1;
						}
						else {
							$rec{'isRole'} = 0;
						}
					}
					elsif($nextline =~ /^\s*FirstName:\s*(.*)$/) {
						$rec{'firstName'} = $1;
					}
					elsif($nextline =~ /^\s*LastName:\s*(.*)$/) {
						$rec{'lastName'} = $1;
					}
					elsif($nextline =~ /^\s*MiddleName:\s*(.*)$/) {
						$rec{'middleName'} = $1;
					}
					elsif($nextline =~ /^\s*RoleName:\s*(.*)$/) {
						$rec{'roleName'} = $1;
					}
					elsif($nextline =~ /^\s*Street:\s*(.*)$/) {
						$rec{"street$nstreet"} = $1;
						$nstreet++;
					}
					elsif($nextline =~ /^\s*City:\s*(.*)$/) {
						$rec{'city'} = $1
					}
					elsif($nextline =~ /^\s*State\/Prov:\s*(.*)$/) {
						$rec{'state'} = $1
					}
					elsif($nextline =~ /^\s*Country:\s*(.*)$/) {
						$rec{'country'} = $1;
					}
					elsif($nextline =~ /^\s*PostalCode:\s*(.*)$/) {
						$rec{'postalCode'} = $1;
					}
					elsif($nextline =~ /^\s*RegDate:\s*(.*)$/) {
						if($1 ne '') {
							$rec{'registerDate'} = $1;
						}
						else {
							$rec{'registerDate'} = '1970-01-01';						
						}
					}
					elsif($nextline =~ /^\s*Updated:\s*(.*)$/) {
						$rec{'updateDate'} = $1;
					}
					elsif($nextline =~ /^\s*Mailbox:\s*(.*)$/) {
						$rec{'mailbox'} = $1;
					}
					elsif($nextline =~ /^\s*OfficePhone:\s*(.*)$/) {
						$rec{'officePhone'} = $1;
					}
					elsif($nextline =~ /^\s*Comment:\s*(.*)$/) {
						$rec{'comment'} = $1 if $ncomment == 0;
						$rec{'comment'} .= " ". $1 if $ncomment > 0;
						$ncomment++;
					}
					elsif($nextline =~ /^\s*Source:\s*(.*)$/) {
						$rec{'source_string'} = $1;
						if($1 eq 'ARIN') {
							$rec{'source'} = 1;
						}
						elsif($1 eq 'RIPE') {
							$rec{'source'} = 2;
						}
						elsif($1 eq 'APNIC') {
							$rec{'source'} = 3;
						}
						else { # unknown
							$rec{'source'} = 0;
						}
					}
					
					# advance to the next line
					$nextline = <INPUT>;
					$lineno++;
		#			print "reading next line: $nextline";
				}
				
				$record++;
				
				$log->log(level=>'error', message=>"ERROR: Record data not parsed properly for last POC\n") if !defined($rec{'pocHandle'});
								
				my $found_record=0;
				if(defined($rec{'pocHandle'}) and !$skipPOC) {
					# update/insert into database
					 $select_from_poc_by_pochandle_sth->execute($rec{'pocHandle'}) or die "Can't select record: $DBI::errstr";;
					 while(my($id) = $select_from_poc_by_pochandle_sth->fetchrow_array()) {
						$found_record=1;
						$record_id = $id;
					 }
					 
					 $select_from_poc_by_pochandle_sth->finish();
					 
					 if($found_record) {
						
						$log->log(level=>'debug', message=>"Found POC record: ". $rec{'pocHandle'} ." in database -- updating id=". $record_id ."\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;
						$update_poc_sth->execute($rec{'pocHandle'}, $rec{'isRole'}, $rec{'firstName'}, $rec{'lastName'}, $rec{'middleName'}, $rec{'roleName'},
												 $rec{'street1'}, $rec{'street2'}, $rec{'street3'}, $rec{'street4'}, $rec{'street5'}, $rec{'street6'},
												 $rec{'city'}, $rec{'state'}, $rec{'country'}, $rec{'postalCode'}, $rec{'registerDate'}, $rec{'comment'}, $rec{'updateDate'}, 
												 $rec{'officePhone'}, $rec{'mailbox'}, $rec{'source'}, $cacheDate, 
												 $record_id)
								or die "Can't select record: $DBI::errstr";
						$update_poc_sth->finish();
					 }
					 else {
						$log->log(level=>'debug', message=>"Found POC record: ". $rec{'orgName'} ." but not in database -- inserting ...\n") if $DEBUG eq 'true' and $opt{verbose} >= 2;
						$insert_into_poc_sth->execute($rec{'pocHandle'}, $rec{'isRole'}, $rec{'firstName'}, $rec{'lastName'}, $rec{'middleName'}, $rec{'roleName'},
												      $rec{'street1'}, $rec{'street2'}, $rec{'street3'}, $rec{'street4'}, $rec{'street5'}, $rec{'street6'},
												      $rec{'city'}, $rec{'state'}, $rec{'country'}, $rec{'postalCode'}, $rec{'registerDate'}, $rec{'comment'}, $rec{'updateDate'}, 
												      $rec{'officePhone'}, $rec{'mailbox'}, $rec{'source'}, $cacheDate, $cacheDate) or die "Can't select record: $DBI::errstr";
						
						$insert_into_poc_sth->finish();
					 }
				}
			
				$log->log(level=>"info", message=>"$record POC records processed from $lineno lines\n") if $opt{'verbose'} >= 1 and $lineno % $DEFAULT_RECORD_DISPLAY_CHUNK_SIZE == 0;

				# commit the records to the database every N records
				if(!defined($opt{'display-only'}) and $record % $DEFAULT_RECORD_DB_CHUNK_SIZE == 0) {
					$dbh->commit() or warn "Can't commit records: $DBI::errstr";
				}
			} # end if parser
		} # end while
		
		# commit one last time
		if(!defined($opt{'display-only'})) {
			$dbh->commit() or warn "Can't commit records: $DBI::errstr";
		}
		
	} # end if
}


sub getDateTimeFormat($) {
	return Time::Format::time_format('Mon dd yyyy hh:mm{in}:ss tz', shift);
}

sub resolve_hostname($)
{
  	my $hostname = shift;
  	use Net::DNS;
  	my $res   = Net::DNS::Resolver->new;
  	my $query = $res->search($hostname);
	my $c =0;
	my $addr;
   	if ($query) 
	{
        	foreach my $rr ($query->answer) 
		{
			next unless $rr->type eq "A";
			$addr = $rr->address if $c == 0;
			$c++;
		}
	} 
	else 
	{
		$addr = 'no address';
	}
	return $addr;
}


# convert quaddot to decimal integer
sub ipv4_quaddot_to_decimal($) {
	my $ip = shift;
	warn "invalid ip address provided" if !defined($ip);
	return -1 if !defined($ip);
	warn "invalid ip address provided: $ip" if(! $ip =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/);
	return "" if(! $ip =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/);
       
	return unpack "N", pack "CCCC", split /\./, $ip;
}

# convert decimal integer to quaddot
sub ipv4_decimal_to_quaddot($) {
	my $decimal = shift;
	warn "invalid decimal value provided" if !defined($decimal);
	return "" if !defined($decimal);
	
	warn "invalid decimal value provided: $decimal" if($decimal < 0 or $decimal > 4294967295);
	return "" if($decimal < 0 or $decimal > 4294967295);
        return join ".", unpack "CCCC", pack "N", $decimal;
}

my $ip_rgx = "\\d+\\.\\d+\\.\\d+\\.\\d+";

# Given an IPv4 address in host, ip/netmask or cidr format
# returns a ip / cidr pair.
sub ipv4_parse($;$) {
  my ($ip,$msk);
  # Called with 2 args, assume first is IP address
  if ( defined $_[1] ) {
    $ip = $_[0];
    $msk= $_[1];
  } else {
    ($ip)  = $_[0] =~ /($ip_rgx)/o;
    ($msk) = $_[0] =~ m!/(.+)!o;
  }

  # Remove white spaces
  $msk =~ s/\s//g if defined $msk;

  # Check Netmask to see if it is a CIDR or Network
  if (defined $msk ) {
    if ($msk =~ /^\d{1,2}$/) {
      # Check cidr
      warn ": invalid cidr: ". $msk ."\n"
        if $msk < 0 or $msk > 32;
    } elsif ($msk =~ /^$ip_rgx$/o ) {
      $msk = ipv4_msk2cidr($msk);
    } else {
      warn ": invalid netmask specification: ". $msk ."\n";
    }
  } else {
    # Host
    return $ip;
  }
  wantarray ? ($ip,$msk) : "$ip/$msk";
}

sub ipv4_dflt_netmask($) {
  my ($ip) = ipv4_parse($_[0]);

  my ($b1) = split /\./, $ip;

  return "255.0.0.0"	if $b1 <= 127;
  return "255.255.0.0"	if $b1 <= 191;
  return "255.255.255.0";
}

# Transform a netmask in a CIDR mask length
sub ipv4_msk2cidr($) {
  my $msk = $_[0];
  my @bytes = split /\./, $msk;
  my $cidr = 0;
  for (@bytes) {
    my $bits = unpack( "B*", pack( "C", $_ ) );
    $cidr +=  $bits =~ tr /1/1/;
  }
  return $cidr;
}

 
my $app = new Pwhoisd();
if(defined($opt{'import-whois-dump'})) {
	$app->readWhoisUpdate();
}
else {
	$app->readBgpUpdate();
}

1;
