#!/usr/bin/perl -w
# This script (sophomorix-belwue) is maintained by Rüdiger Beck
# It is Free Software (License GPLv3)
# If you find errors, contact the author
# jeffbeck@web.de  or  jeffbeck@linuxmuster.net

# modules
use strict;
#use Quota;
use Getopt::Long;
#use IMAP::Admin;
#use DBI;
#use utf8;
use HTML::TableExtract;
use Net::LDAP;
use JSON;
use Data::Dumper;
$Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Useqq = 1;
$Data::Dumper::Terse = 1; 

use Sophomorix::SophomorixBase qw(
                                 print_line
                                 print_title
                                 log_script_start
                                 log_script_end
                                 log_script_exit
                                 check_options
                                 config_sophomorix_read
                                 result_sophomorix_init
                                 result_sophomorix_add
                                 result_sophomorix_check_exit
                                 result_sophomorix_print
                                 );

use Sophomorix::SophomorixSambaAD qw(
                                 AD_bind_admin
                                 AD_unbind_admin
                                 AD_dns_get
                                    );

my @arguments = @ARGV;

# ===========================================================================
# Optionen verarbeiten
# ==========================================================================

# Variablen für Optionen
$Conf::log_level=1;
my $help=0;
my $info=0;
my $json=0;
my $ldap_fetch=1;
my $no_ldap_fetch=0;
my $copy_to_teachershare=0;
my $create_login_data="";
my $dump_belwue_data=0;
my $parse_belwue_data=0;

my $config_file="/etc/linuxmuster/sophomorix/belwue-mail/belwue.conf";
my $belwue_downloads="/var/lib/sophomorix/belwue/downloads";
my $belwue_mailboxes="/var/lib/sophomorix/belwue/belwue.multimailboxes";
my $belwue_maillist="/var/lib/sophomorix/belwue/belwue.maillists";
my $belwue_aliases="/var/lib/sophomorix/belwue/belwue.aliases";

# copy-to-teachershare
my $targetdir="/var/lib/sophomorix/belwue-copy-to-teachershare";
my $targetsubdir="/var/lib/sophomorix/belwue-copy-to-teachershare/logindata";

system("mkdir -p /var/lib/sophomorix/belwue"); 

# Parsen der Optionen
my $testopt=GetOptions(
           "help|h" => \$help,
           "info|i" => \$info,
           "json|j+" => \$json,
           "verbose|v+" => \$Conf::log_level,
           "dump-belwue-data" => \$dump_belwue_data,
           "parse-belwue-data" => \$parse_belwue_data,
           "create-login-data=s" => \$create_login_data,
           "copy-to-teachershare" => \$copy_to_teachershare,
          );

my %sophomorix_result=&result_sophomorix_init("sophomorix-belwue");
# Prüfen, ob Optionen erkannt wurden, sonst Abbruch
&check_options($testopt);

my %config=&read_config($config_file);
my %belwue=();
my @accounts_short=();

my @multi_mailboxes=();
my @aliases=();
my @maillists=();
my @groups=();

# calculate connect data
$config{'connect'}{BASE}="CN=".$config{'config'}{'BASE'};
$config{'connect'}{BASESIMPLE}=$config{'config'}{'BASE'};
$config{'connect'}{SCOPE}="sub";
$config{'connect'}{FILTER}="(cn=*)";
$config{'connect'}{HOST}=$config{'config'}{PROTOCOL}."://".
                         $config{'config'}{SERVER}.":".
                         $config{'config'}{PORT};
$config{'connect'}{ADMINUSER}=$config{'config'}{ADMINUSER};
$config{'connect'}{ADMINURL}=$config{'config'}{ADMINURL};
$config{'connect'}{PASSWORD}=$config{'config'}{PASSWORD};

my ($adminuser,$maildomain)=split(/\@/,$config{'config'}{ADMINUSER});
$config{'connect'}{MAILDOMAIN}=$maildomain;

#print Dumper(%config);
#exit;

# --help
if ($help==1) {
   # Scriptname ermitteln
   my @list = split(/\//,$0);
   my $scriptname = pop @list;
   # Befehlsbeschreibung
   print('
sophomorix-belwue

Options
  -h  / --help
  -i  / --info
  -v  / --verbose
  -vv / --verbose --verbose

  --dump-belwue-data (download and parse)
  --parse-belwue-data (parse already downloaded data)
  --create-login-data /path/to/file
  --copy-to-teachershare

Please see the sophomorix-belwue(8) man pages for full documentation
');
   print "\n";
   exit;
}



# --info
if ($info==1) {


}

##################################################
# Script starts here
##################################################

# --dump-belwue-data
if ($dump_belwue_data==1 or $parse_belwue_data){
    ############################################################
    # ldap address book
    ############################################################ 
    print "\n";
    print "============================================================\n";
    print "Accessing belwue address book:\n";
    my ($ldap) = &bind_admin();
    if($Conf::log_level>=2){
        print "  Searching for data in Belwue address book via ldap\n";
        print "    SEARCHBASE: $config{'connect'}{BASE}\n";
        print "    SCOPE: $config{'connect'}{SCOPE}\n";
        print "    FILTER: $config{'connect'}{FILTER}\n";
    }
    my $mesg = $ldap->search(
                   base   => $config{'connect'}{BASE},
                   scope => $config{'connect'}{SCOPE},
                   filter => $config{'connect'}{FILTER},
                        );

    my $max = $mesg->count; 
    my $count = $mesg->count;
    for( my $index = 0 ; $index < $max ; $index++) {
        my $num=$index+1;
        my $entry = $mesg->entry($index);
        my $dn=$entry->dn();
        my $uid=$entry->get_value('uid');
        my $cn=$entry->get_value('cn');
        my $given=$entry->get_value('givenName');
        my $sn=$entry->get_value('sn');
        my $server=$entry->get_value('hostServer');
        my $mail=$entry->get_value('mail');

        if (not defined $given){
            $given="";
        }
        if($Conf::log_level>=3){
	    print "$num) dn: $dn\n";
	    print "   * cn:          $cn\n";
	    print "   * givenName:   $given\n";
	    print "   * hostServer:  $server\n";
	    print "   * mail:        $mail\n";
	    print "   * sn:          $sn\n";
	    print "   * uid:         $uid\n";
            print "\n";
        }

        $belwue{'objects'}{$uid}{'CN'}=$cn;
        $belwue{'objects'}{$uid}{'givenName'}=$given;
        $belwue{'objects'}{$uid}{'hostServer'}=$server;
        $belwue{'objects'}{$uid}{'mail'}=$mail;
        $belwue{'objects'}{$uid}{'sn'}=$sn;
        $belwue{'objects'}{$uid}{'uid'}=$uid;


    }
        #print Dumper(%belwue);

    print "$max entries found via ldaps\n";
    print "============================================================\n";
    &unbind_admin($ldap); 
    ############################################################
    # downloading objects
    ############################################################ 
    if ($parse_belwue_data==0){
      print "Downloading and parsing html files:\n";
      system("mkdir -p $belwue_downloads");

      # object list
      my $download_objects_command="wget ".
          "$config{'config'}{WGET_OPTIONS} ".
          "-q --http-user=admin ".
          "--http-password=$config{'connect'}{PASSWORD} ".
          "--output-document=$belwue_downloads/objects.html ".
          "\"https://$config{'connect'}{ADMINURL}/$config{'connect'}{BASESIMPLE}".
          "/ObjectList.html?domainName=$config{'connect'}{BASESIMPLE}\"";

      if($Conf::log_level>=2){
          print "\nWGET: $download_objects_command\n\n";
      }
      my $return=system($download_objects_command);
      &wget_return($return,"Objects","$belwue_downloads/objects.html");
    }

    ############################################################
    # parsing objects
    ############################################################

    # file to parse
    my $html_doc = "$belwue_downloads/objects.html";
    print "   * Parsing $html_doc:\n\n";

    # list of headers
    my $table_headers = [ 'Objekt', 'Typ','Speicher','Letzter Zugriff' ];

    # constructor
    my $table_extract = HTML::TableExtract->new(headers => $table_headers);
    $table_extract->utf8_mode(1);
    $table_extract->parse_file($html_doc);
    my ($table) = $table_extract->tables;

    foreach my $row ($table->rows) {
        my ($object,$type,$storage,$access)=@$row;
        if (not defined $storage){
            $storage="none";
        } else {
	    if ($type eq "Multi-Mailbox"){
		$belwue{'objects'}{$object}{'quota_used'}=$storage;
	    } elsif ($type eq "Alias") {
		$belwue{'objects'}{$object}{'uid'}=$storage;
	    } elsif ($type eq "Gruppe") {
		$belwue{'objects'}{$object}{'member_count'}=$storage;
	    }
	}
        if (not defined $access){
            $access="never";
        } else {
	    if ($type eq "Multi-Mailbox"){
		$belwue{'objects'}{$object}{'last_access'}=$access;
	    } elsif ($type eq "Alias") {
                # nothing to do
	    } elsif ($type eq "Gruppe") {
                # nothing to do
	    }
	}
	print "   * Extracted row: $object  $type  $storage  $access\n";
        $belwue{'objects'}{$object}{'type'}=$type;
        # add to types
        if ($type eq "Mailingliste"){
            $belwue{'maillist'}{$object}{'type'}=$type;
            push @maillists, $object;
            ############################################################
            # downloading maillist data
            ############################################################
            if ($parse_belwue_data==0){
                my $download_list_command="wget ".
                    "$config{'config'}{WGET_OPTIONS} ".
                    "-q --http-user=admin ".
                    "--http-password=$config{'connect'}{PASSWORD} ".
                    "--output-document=$belwue_downloads/$object.html ".
                    "\"https://$config{'connect'}{ADMINURL}/".
                    "$config{'connect'}{BASESIMPLE}".
                    "/Subscribers.html?InCluster=1&domainName=".
                    "$config{'connect'}{BASESIMPLE}&&listName=$object\"";
                if($Conf::log_level>=2){
                    print "\nWGET: $download_list_command\n\n";
                }
                my $return=system($download_list_command);
                &wget_return($return,$object,"$belwue_downloads/$object.html");
	    }

            ############################################################
            # parsing maillist data
            ############################################################
            my $html_doc = "$belwue_downloads/$object.html";
            print "   * Parsing $html_doc\n";

            # list of headers
            # Umlaut in header does nor work
            my $table_headers = [ 'Email',
                                  'Zustellungsart',
                                  'Anmeldezeit',
                                  'Nachrichten'];
            # constructor
            my $table_extract = HTML::TableExtract->new(headers => $table_headers);
            $table_extract->utf8_mode(1);
            $table_extract->parse_file($html_doc);
            my ($table) = $table_extract->tables;
	    my $row_count=0;
            my $empty_row=0;
            foreach my $row ($table->rows) {
		$row_count++;
                my ($member_email,$type,$time,$bounced,$displayname)=@$row;
		if (not defined $member_email){
		    $empty_row=1;
                    next;
		}
                my ($uid,$domain)=split(/\@/,$member_email);

                if ($type ne "Abonnement aufheben"){
                    $belwue{'maillist'}{$object}{'members'}{$member_email}="member";
                    if ($domain eq $config{'connect'}{MAILDOMAIN}){
			$belwue{'objects'}{$uid}{'maillist_membership'}{$object}=$type;
			$belwue{'objects'}{$object}{'members'}{$uid}=$type;
		    } else {
			# foreign domain, nothing to do
		    }
                    push @{ $belwue{'maillist'}{$object}{'memberlist'} }, $member_email;
                } else {
		    $belwue{'warnings'}{$uid}{'maillist_membership_warning'}{$object}=$type;
		}
		$belwue{'objects'}{$uid}{'maillist_membership_created'}{$object}=$time;
		$belwue{'objects'}{$uid}{'maillist_messages_bounced'}{$object}=$bounced;
            }
	    if ($empty_row==0){
		print "   * $row_count rows parsed\n";
	    } else {
		print "   * $row_count empty row parsed\n";
	    }
            # sort the memberlist if not empty
            if ( not  $#{ $belwue{'maillist'}{$object}{'memberlist'} }==-1 ){
                 @{ $belwue{'maillist'}{$object}{'memberlist'} } = sort @{ $belwue{'maillist'}{$object}{'memberlist'}  };
            }
        } elsif ($type eq "Gruppe"){
            $belwue{'group'}{$object}{'type'}=$type;
            push @groups, $object;
            ############################################################
            # downloading group data
            ############################################################
            if ($parse_belwue_data==0){
                my $download_list_command="wget ".
                    "$config{'config'}{WGET_OPTIONS} ".
                    "-q --http-user=admin ".
                    "--http-password=$config{'connect'}{PASSWORD} ".
                    "--output-document=$belwue_downloads/$object.html ".
                    "\"https://$config{'connect'}{ADMINURL}/".
                    "$config{'connect'}{BASESIMPLE}".
                    "/Group.html?&domainName=".
                    "$config{'connect'}{BASESIMPLE}&&groupName=$object\"";
                if($Conf::log_level>=2){
                    print "\nWGET: $download_list_command\n\n";
                }
                my $return=system($download_list_command);
                &wget_return($return,$object,"$belwue_downloads/$object.html");
	    }

            ############################################################
            # parsing group data
            ############################################################
            my $html_doc = "$belwue_downloads/$object.html";
            print "   * Parsing $html_doc\n";

            my $string=`cat $html_doc | grep input | grep Members | grep value`;
            my (@lines)=split(/\n/,$string);
	    foreach my $line (@lines){
		my ($unused1,$used)=split("value=",$line);
		my ($member_mail,$unused2)=split(" ",$used);
		$member_mail=~s/"//g;
                print "<$member_mail>\n";
		if ($member_mail ne ""){
                    my ($uid,$domain)=split(/\@/,$member_mail);
		    if (not defined $domain or
                        $domain eq $config{'connect'}{MAILDOMAIN}
                       ){
                        $belwue{'objects'}{$uid}{'group_membership'}{$object}=$type;
                        $belwue{'objects'}{$object}{'members'}{$uid}="member";
                    } else {
                        $belwue{'objects'}{$object}{'members'}{$member_mail}="member";
                    }
		}
	    }
        } elsif ($type eq "Alias"){
            $belwue{'alias'}{$object}=$storage;
            $belwue{'alias_reverse'}{$storage}=$object;
            # aliases are in storage
            $belwue{'objects'}{$storage}{'alias'}=$object;
            push @aliases, $object;
        } elsif ($type eq "Multi-Mailbox"){
            $belwue{'multimailbox'}{$object}{'type'}="Multi-Mailbox";
            $belwue{'multimailbox'}{$object}{'storage'}=$storage;
            $belwue{'multimailbox'}{$object}{'access'}=$access;
            push @multi_mailboxes, $object;
        } else {
            print "Unknown object $type\n";
        }
	print "\n";
    }
    ############################################################
    # printout data/write data into files
    ############################################################
    open (BOX, ">$belwue_mailboxes");
    open (ALIAS, ">$belwue_aliases");
    open (LIST, ">$belwue_maillist");

    @multi_mailboxes = sort @multi_mailboxes;
    print "\nMultiMailboxen:\n";
    my $num_mbox=1;
    foreach my $uid (@multi_mailboxes){
        print BOX "$uid\n";
        print "   $num_mbox) $uid\n";
        $num_mbox++;
    }

    @maillists = sort @maillists;
    print "\nMailing Listen:\n";
    foreach my $list (@maillists){
        my $member_string=join(",",@{ $belwue{'maillist'}{$list}{'memberlist'} });
        print LIST "$list:$member_string\n";
        print "   * $list\n";
        my $num_list=1;
        foreach my $member ( @{ $belwue{'maillist'}{$list}{'memberlist'} } ){
            if (not defined $member){
                next;
            } 
            print "     $num_list) --> $member\n";
            $num_list++;
        }
    }

    @aliases = sort @aliases;
    print "\nMail aliases:\n";
    my $num=1;
    foreach my $alias (@aliases){
        print ALIAS "$alias:$belwue{'alias'}{$alias}\n";
        print "   $num) $alias --> $belwue{'alias'}{$alias}\n";
        $num++
    }

    close(BOX);
    close(ALIAS);
    close(LIST);

    if($Conf::log_level>=2){
        print Dumper(\%belwue);
        my $ref_belwue=\%belwue;
    }
} 




# --create-login-data
if($create_login_data ne ""){
    # remove old data        
    system("rm -rf $targetdir"); 
    system("mkdir -p $targetdir"); 

    open(FILE,"$create_login_data") || 
              die "ERROR: $create_login_data not found!";
    while(<FILE>){
        chomp();
        my ($uid,$realname,$type,$pass,$storage,$alias)=split(/\t/);
        if ($uid eq "Name" and $realname eq "Realname"){
            next;
        }
        print "   * $uid\n";
        system("mkdir -p $targetdir/${uid}");
        open(USER,">$targetdir/${uid}/${uid}-mail-login.txt") || 
              die "ERROR: $targetdir not found!";
        print USER "\n";
        print USER "Zugangsdaten für die Emailadresse: ${uid}\@$config{'config'}{BASE}\n";
        print USER "                            Alias: $alias\@$config{'config'}{BASE}\n";
        print USER "\n";
        #print USER "Belwue-Mail-URL:  https://$config{'config'}{SERVER}\n";
        print USER "Benutzername:     ${uid}\@$config{'config'}{BASE}\n";
        print USER "Erst-Password:    $pass\n";
        print USER "Erst-Mailquota:   $storage\n";
        print USER "Mailbox-Typ:      $type\n";
        print USER "Adressbuch-Name:  $realname\n";
        print USER "\n";
        close(USER);
        system("chown ${uid}.root $targetdir/${uid}");
        system("chmod 0700 $targetdir/${uid}");
        system("chown ${uid}.root $targetdir/${uid}/${uid}-mail-login.txt");
        system("chmod 0600 $targetdir/${uid}/${uid}-mail-login.txt");
    }
    close(FILE);
    print "Login-Data created in $targetdir\n";
}


# --copy-to-teachershare
if($copy_to_teachershare){
	my $copy_command="rsync -av ${targetdir}/ ".
          "$config{'config'}{TEACHERUSER}\@".
          "$config{'config'}{TEACHERSERVER}:".
          "$config{'config'}{TEACHERSERVERDIR}";
    print "$copy_command\n";
    system($copy_command);
}


if($Conf::log_level>=2){
    print Dumper(%config);
}



############################################################
# subs
############################################################

sub bind_admin {
    # bind
    if($Conf::log_level>=2){
        print "  Connecting to Belwue via ldaps\n";
        print "    HOST:      $config{'connect'}{HOST} \n";
        print "    BIND-DN:   $config{'connect'}{ADMINUSER} \n";
        print "    PASSWORD:  $config{'connect'}{PASSWORD}\n";
    }
    my $ldap = Net::LDAP->new($config{'connect'}{HOST}) or
        die "\nERROR: Connection to $config{'connect'}{HOST} failed!\n",
            " USER: $config{'connect'}{ADMINUSER}, PASSWORD: $config{'connect'}{PASSWORD})\n\n";

    my $mesg = $ldap->bind($config{'connect'}{ADMINUSER}, 
                      password => $config{'connect'}{PASSWORD});
    # show errors from bind
    $mesg->code && die $mesg->error;
    return ($ldap);
}



######################################################################
sub unbind_admin {
    my ($ldap) = @_;
    my $mesg = $ldap->unbind();
    #  show errors from unbind
    $mesg->code && die $mesg->error;
}



######################################################################
# error, when options are not given correctly
#sub  check_options{
#   my ($parse_ergebnis) = @_;
#   if (not $parse_ergebnis==1){
#      my @list = split(/\//,$0);
#      my $scriptname = pop @list;
#      print "\nYou have made a mistake, when specifying options.\n"; 
#      print "See error message above. \n\n";
#      print "... $scriptname is terminating.\n\n";
#      exit;
#   } else {
#      if($Conf::log_level>=3){
#         print "All options  were recognized.\n";
#      }
#   }
#}



######################################################################
sub wget_return {
    my ($return,$object,$file)=@_;
    if ($return==0){
        print "   * Succesfully downloded html page for $object: $file\n";
    } else {
        print "   * ERROR downloading $object: wget returned error code $return\n";
        exit;
    }
}



######################################################################
sub read_config {
    my %config=();
    my ($file) = @_;
    if (-e $file) {
        open (SECRET, $file);
        while(<SECRET>){
            if(/^\#/){
                next;
            }
            my ($key,$value)=split(/=/);
            $key=~s/^\s+//g;# remove leading whitespace
            $key=~s/\s+$//g;# remove trailing whitespace
            $value=~s/^\s+//g;# remove leading whitespace
            $value=~s/\s+$//g;# remove trailing whitespace
            $config{'config'}{$key}=$value;
        }
        close(SECRET);
    } else {
        print "Config File $file must be created with:\n\n";
        print "# PASSWORD of the user admin\n";
        print "PASSWORD=xyz\n";
        print "ADMINUSER=admin\@mail.bszleo.de\n";
        print "ADMINURL=admin\@mail.bszleo.de\n";
        print "# SERVER should be mbox1.belwue.de for everybody\n";
        print "SERVER=mbox1.belwue.de\n";
        print "BASE=mail.bszleo.de\n";
        print "# ldap connection:\n";
        print "PORT=636\n";
        print "PROTOCOL=ldaps\n";
        print "WGET_OPTIONS=--no-check-certificate\n";
	print "\n";
        exit;
    }
    return %config;
}

