Das Problem der sicheren Batchübertragung von Daten hat nicht nuzr der Denic.
Ichhabe für diesen Zweck vor Jahren mal das folgende Perl - Skript geschrieben, für eine Übertragung mit zusätzlicher Prüfsumme pro Datei die unmittelbar vor der Übertragung erstellt wird:.
Das Ergebnis für die einzelnen Übertragungen wird zur asynchronen Weiterverarbeitung durch Nagios/Nrpe oder ähnliche Dienste in Status- und Message - Dateien pro Server festgehalten.
use Data::Dumper;
$logfile="/home/vipdta/dta.log";
runtransfer("server1","192.168.150.1","/home/samba/verwaltung/VIP/Dta");
runtransfer("server2","192.168.151.1","/home/Verwaltung/vip/Dta");
#....
###################################
#Function: runtransfer
#Args: $remotename,$remotehost,$remotepath
sub runtransfer {
$remotename=shift;
$remotehost=shift;
$remotepath=shift;
#create a timestamp for the logbuffer
($date,$time) = &time_stamp();
$logbuffer="$date - $time : $remotename - TRANSFER START \n";
writetofile($logfile,"$logbugffer");
#generate a list of all files to transfer
my ($resdtafilelist,@dtafilelist)=getdtafilestotransfer("$remotename","$remotehost","$remotepath");
#check wether filelist could be created and wether there are files to transfer
if ("$resdtafilelist" eq "0" && "$#dtafilelist" eq "-1"){
#add filelist to logbuffer
push (@logbuffer,"$date - $time : $remotename - TRANSFER PREPARE LIST: NO FILES TO TRANSFER\n");
#write to logfile
writetofile($logfile,"@logbuffer");
###############################################
#nagios - flagfiles: status and message
writetofile2("log/FLAG-$remotename.status",">","0");
writetofile2("log/FLAG-$remotename.msg",">","$date - $time : $remotename - NO FILES TO TRANSFER");
#no files = nothing to do here now - exit this function gracefully
return 0;
}else {
#now do the transfer
($status,$text)=dtatransfer("$remotename","$remotehost","$remotepath","@dtafilelist");
################################################
#nagios - flagfiles: status and message
#check return value
$retval=$?;
print "DEB RETVAL: $retval / $status / $text \n ";
#check return value and set nagios flagfile
if ( "$retval" eq "0" ){
writetofile2("log/FLAG-$remotename.status",">","$status");
writetofile2("log/FLAG-$remotename.msg",">","$text");
} else {
writetofile2("log/FLAG-$remotename.status",">","$status");
writetofile2("log/FLAG-$remotename.msg",">","$text");
}
#create a timestamp for the logbuffer
($date,$time) = &time_stamp();
$logbuffer="$date - $time : $remotename - TRANSFER END \n";
writetofile($logfile,"$logbuffer");
}
}
############################################################
#Function: getdtafilestotransfer
#Args: $remotename,$remotehost,remotepath
#Lists a remote directory and returns a filelist as array
sub getdtafilestotransfer {
$remotename=shift;
$remotehost=shift;
$remotepath=shift;
print "########################################################################################################\n";
print "# Generating list for DTA Filetransfer for $remotename on $remotehost at $remotepath\n";
print Dumper(@dtafilestotransfer);
print "########################################################################################################\n";
#list all files in remote Dta directory
$dtafiles_remote=`ssh vipdta\@$remotehost ls -D $remotepath`;
$resdtafiles_remote=checkresult($?);
@dtafiles_remote=split(' ',$dtafiles_remote);
#list all files in local Dta directory - from file
#$transferlog="/home/vipdta/Dta-Transferlog";
#open(DAT, $transferlog) || die("Could not open $transferlog!");
#@dtafiles_local=;
#close(DAT);
#create Transferlist
my $i=0;
my @dtafilestotransfer;
for my $dtafile (@dtafiles_remote){
unless (grep $dtafile eq $_, @dtafiles_local) {
print "$dtafile has not yet been transferred, added to transfer list.\n";
push(@dtafilestotransfer,$dtafile);
}
}
#create a timestamp for the logbuffer
($date,$time) = &time_stamp();
#check results and fill logbuffer
if ( "$resdtafiles_remote" eq "ok" ) {
push (@logbuffer,"$date - $time : $remotename - TRANSFER PREPARE SUCCESS: Filelist created from $remotehost:/$remotepath $resdtafiles_remote.\n");
$TRANSFERPREPARE=0;
} else {
push (@logbuffer,"$date - $time : $remotename - TRANSFER PREPARE FAILURE: Filelist not created from $remotehost:/$remotepath $resdtafiles_remote.\n");
$TRANSFERPREPARE=1;}
#add filelist to logbuffer
push (@logbuffer,"$date - $time : $remotename - TRANSFER PREPARE LIST: @dtafilestotransfer \n");
#write to logfile
writetofile($logfile,"@logbuffer");
#return the resultstatus and the resultarray
return ($TRANSFERPREPARE,@dtafilestotransfer);
}
#############################################################
#Function: dtatransfer
#Args: $remotename,$remotehost,$remotepath,@dtafilestotranfer
sub dtatransfer {
#Do a PARANOID - SAFE Transfer:
# for each file in transferlist:
# -create MD5SUMS on file
# -transfer the file
# -create MD5SUM on transferred file
# -check MD5SUM
# -try a retransfer on checksum mismatch and log error
$remotename=shift;
$remotehost=shift;
$remotepath=shift;
@dtafilestotransfer=split(" ",shift);
my @logbuffer;
print "##########################################################\n";
print "# Starting DTA Transfer for $remotename from $remotehost \n";
print Dumper(@dtafilestotransfer);
print "##########################################################\n";
for my $dtafile (@dtafilestotransfer) {
#md5sum
$cmd="ssh vipdta\@$remotehost md5sum $remotepath/$dtafile";
my $md5sumtmp=`$cmd`;
$resmd5remote=checkresult($?);
my @md5sumremote=split(" ",$md5sumtmp);
#transfer
$cmd="scp vipdta\@$remotehost:$remotepath/$dtafile /home/vipdta/Dta";
my $transfer=`$cmd`;
$restransfer=checkresult($?);
print "$dtafile - Uebertragung: $res \n";
#md5sum of transferred file
$cmd="md5sum /home/vipdta/Dta/$dtafile";
my $md5sumtmp=`$cmd`;
$resmd5local=checkresult($?);
my @md5sumlocal=split(" ",$md5sumtmp);
#create a timestamp for the logbuffer
($date,$time) = &time_stamp();
#ok, do the checks
print "MD5LOCAL $md5sumlocal[0] MD5REMOTE $md5sumremote[0] - $dtafile \n";
if ( "$md5sumremote[0]" eq "$md5sumremote[0]" ){
print "Transfer of $dtafile successfull, MD5 is clear. \n";
$ISTRANSFEROK="0";
push (@logbuffer,"$date - $time : $remotename - $dtafile TRANSFER SUCCESS: MD5REMOTE:$resmd5remote TRANSFER:$restransfer MD5LOCAL:$resmd5local
\n");
} else {
print "Error: Transfer of $dtafile failed, MD5 mismatch. \n";
$ISTRANSFEROK="1";
push (@logbuffer,"$date - $time : $remotename - $dtafile TRANSFER FAILURE: MD5REMOTE:$resmd5remote TRANSFER:$restransfer MD5LOCAL:$resmd5local\n");
}
#now check wether it is save to delete the files on the remote side
if ( "$ISTRANSFEROK" eq "0") {
$cmd="ssh vipdta\@$remotehost rm $remotepath/$dtafile";
my $deletedtafile=`$cmd`;
$resdeleted=checkresult($?);
#create a timestamp for the logbuffer
($date,$time) = &time_stamp();
#fill the logbuffer
push (@logbuffer,"$date - $time : $remotename - $dtafile TRANSFER END SUCCESS: $dtafile on $remotehost deleted.\n");
} else {
print "$dtafile could not be successfully transferred \n";
$cmd="echo Dta/$dtafile";
my $deleteddtafile=`$cmd`;
push (@logbuffer,"$date - $time : $remotename - $dtafile TRANSFER END FAILURE: MD5MISMATCH, local $dtafile deleted.\n");
}
}
writetofile($logfile,"@logbuffer");
return ($ISTRANSFEROK,"@logbuffer");
}
##########################################
#Function: checkresult
#check return codes for ssh/scp and md5sum
sub checkresult {
my $res=$_;
if ( $res == 0 ) {
print "ok ";
return "ok";} elsif ( $res == 1) {
print "error ";
return "error";
} elsif ( $res == 256 ){
print "error: File not found or Server Timeout ";
return "error: File not found";
} else {
print "unknown error";
return "unknown error";
}
}
#########################
#Function: writetofile
#Args: $file,@content
sub writetofile {
my $file=shift;
my $text=shift;
#my @logelems=split(" ",shift);
open(FILE, ">>$file");
print FILE "$text \n";
close FILE;
}
#########################
#Function: writetofile2
#Args: $file,$mode,@content
sub writetofile2 {
my $file=shift;
my $mode=shift;
my $text=shift;
#my @logelems=split(" ",shift);
open(FILE, "$mode$file");
print FILE "$text \n";
close FILE;
}
#############################
#Function:timestamp
#create a timestamp
#Args: void, returns ($date,$time)
sub time_stamp {
my ($d,$t);
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
$year += 1900;
$mon++;
$d = sprintf("%4d-%2.2d-%2.2d",$year,$mon,$mday);
$t = sprintf("%2.2d:%2.2d:%2.2d",$hour,$min,$sec);
return($d,$t);
}
