#!/usr/bin/env python
#
# svn-backup-dumps.py -- Create dumpfiles to backup a subversion repository.
#
# ====================================================================
# Copyright (c) 2006 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
# This software consists of voluntary contributions made by many
# individuals.  For exact contribution history, see the revision
# history and logs, available at http://subversion.tigris.org/.
# ====================================================================
#
# This script creates dump files from a subversion repository on *IX.
# It is intended for use in cron jobs and post-commit hooks.
#
# The basic operation modes are:
#    1. Create a full dump (revisions 0 to HEAD).
#    2. Create incremental dumps containing at most N revisions.
#    3. Create incremental single revision dumps (for use in post-commit).
#
# All dump files are prefixed with the basename of the repository. All
# examples below assume that the repository '/srv/svn/repos/src' is
# dumped so all dumpfiles start with 'src'.
#
# Optional functionality:
#    4. Create gzipped dump files.
#    5. Create bzipped dump files.
#    6. Transfer the dumpfile to another host using ftp.
#    7. Transfer the dumpfile to another host using smb.
#
# See also 'svn-backup-dumps.py -h'.
#
#
# 1. Create a full dump (revisions 0 to HEAD).
#
#    svn-backup-dumps.py <repos> <dumpdir>
#
#    <repos>      Path to the repository.
#    <dumpdir>    Directory for storing the dump file.
#
#    This creates a dump file named 'src.000000-NNNNNN.svndmp.gz'
#    where NNNNNN is the revision number of HEAD.
#
# 2. Create incremental dumps containing at most N revisions.
#
#    svn-backup-dumps.py -c <count> <repos> <dumpdir>
#
#    <count>      Count of revisions per dump file.
#    <repos>      Path to the repository.
#    <dumpdir>    Directory for storing the dump file.
#
#    When started the first time with a count of 1000 and if HEAD is
#    at 2923 it creates the following files:
#
#    src.000000-000999.svndmp.gz
#    src.001000-001999.svndmp.gz
#    src.002000-002923.svndmp.gz
#
#    Say the next time HEAD is at 3045 it creates these two files:
#
#    src.002000-002999.svndmp.gz
#    src.003000-003045.svndmp.gz
#
#
# 3. Create incremental single revision dumps (for use in post-commit).
#
#    svn-backup-dumps.py -r <revnr> <repos> <dumpdir>
#
#    <revnr>      A revision number.
#    <repos>      Path to the repository.
#    <dumpdir>    Directory for storing the dump file.
#
#    This creates a dump file named 'src.NNNNNN.svndmp.gz' where
#    NNNNNN is the revision number of HEAD.
#
#
# 4. Create gzipped dump files.
#
#    svn-backup-dumps.py -z ...
#
#    ...          More options, see 1-3, 6, 7.
#
#
# 5. Create bzipped dump files.
#
#    svn-backup-dumps.py -b ...
#
#    ...          More options, see 1-3, 6, 7.
#
#
# 6. Transfer the dumpfile to another host using ftp.
#
#    svn-backup-dumps.py -t ftp:<host>:<user>:<password>:<path> ...
#
#    <host>       Name of the FTP host.
#    <user>       Username on the remote host.
#    <password>   Password for the user.
#    <path>       Subdirectory on the remote host.
#    ...          More options, see 1-5.
#
#    If <path> contains the string '%r' it is replaced by the
#    repository name (basename of the repository path).
#
#
# 7. Transfer the dumpfile to another host using smb.
#
#    svn-backup-dumps.py -t smb:<share>:<user>:<password>:<path> ...
#
#    <share>      Name of an SMB share in the form '//host/share'.
#    <user>       Username on the remote host.
#    <password>   Password for the user.
#    <path>       Subdirectory of the share.
#    ...          More options, see 1-5.
#
#    If <path> contains the string '%r' it is replaced by the
#    repository name (basename of the repository path).
#
#
#
# TODO:
#  - find out how to report smbclient errors
#  - improve documentation
#

__version = "0.4"

import sys
import os
import fcntl
import select
import gzip
import os.path
from optparse import OptionParser
from ftplib import FTP

try:
    import bz2
    have_bz2 = True
except ImportError:
    have_bz2 = False


class Popen24Compat:

    def __init__( self, args, bufsize=0, executable=None, stdin=None,
            stdout=None, stderr=None, preexec_fn=None, close_fds=False,
            shell=False, cwd=None, env=None, universal_newlines=False,
            startupinfo=None, creationflags=0 ):

        if isinstance( args, list ):
            args = tuple( args )
        elif not isinstance( args, tuple ):
            raise RipperException, "Popen24Compat: args is not tuple or list"

        self.stdin = None
        self.stdout = None
        self.stderr = None
        self.returncode = None

        if executable == None:
            executable = args[0]

        if stdin == PIPE:
            stdin, stdin_fd = os.pipe()
            self.stdin = os.fdopen( stdin_fd )
        elif stdin == None:
            stdin = 0
        else:
            stdin = stdin.fileno()
        if stdout == PIPE:
            stdout_fd, stdout = os.pipe()
            self.stdout = os.fdopen( stdout_fd )
        elif stdout == None:
            stdout = 1
        else:
            stdout = stdout.fileno()
        if stderr == PIPE:
            stderr_fd, stderr = os.pipe()
            self.stderr = os.fdopen( stderr_fd )
        elif stderr == None:
            stderr = 2
        else:
            stderr = stderr.fileno()

        # error pipe
        err_read, err_write = os.pipe()
        fcntl.fcntl( err_write, fcntl.F_SETFD, fcntl.FD_CLOEXEC )

        self.pid = os.fork()
        if self.pid < 0:
            raise Exception, "Popen24Compat: fork"
        if self.pid == 0:
            # child
            os.close( err_read )
            fcntl.fcntl( err_write, fcntl.F_SETFD, fcntl.FD_CLOEXEC )
            if self.stdin:
                self.stdin.close()
            if self.stdout:
                self.stdout.close()
            if self.stderr:
                self.stderr.close()
            if stdin != 0:
                os.dup2( stdin, 0 )
                os.close( stdin )
            if stdout != 1:
                os.dup2( stdout, 1 )
                os.close( stdout )
            if stderr != 2:
                os.dup2( stderr, 2 )
                os.close( stderr )
            try:
                if shell:
                    # should spawn a shell here...
                    os.execvp( executable, args )
                else:
                    os.execvp( executable, args )
            except:
                err = sys.exc_info()[1]
            # exec error
            os.write( err_write, str(err) )
            os._exit( 255 )
        else:
            # parent
            os.close( err_write )
            if stdin != 0:
                os.close( stdin )
            if stdout != 0:
                os.close( stdout )
            if stderr != 0:
                os.close( stderr )
            sr, sw, se = select.select( [ err_read ], [], [ err_read ] )
            if len( se ) == 1:
                os.close( err_read )
                raise Exception, "Popen24Compat: err pipe read error"
            if len( sr ) == 1:
                err = os.read( err_read, 1024 )
            os.close( err_read )
            if len( err ) != 0:
                raise Exception, "Popen24Compat: exec error: " + err

    def poll( self ):
        self.__wait( os.WNOHANG )
        return self.returncode

    def wait( self ):
        self.__wait( 0 )
        return self.returncode

    def __wait( self, options ):
        pid, rc = os.waitpid( self.pid, options )
        if pid != 0:
            self.returncode = rc

def PopenConstr( args, bufsize=0, executable=None, stdin=None, stdout=None,
            stderr=None, preexec_fn=None, close_fds=False, shell=False,
            cwd=None, env=None, universal_newlines=False, startupinfo=None,
            creationflags=0 ):
    return Popen24Compat( args, bufsize=bufsize, executable=executable,
                stdin=stdin, stdout=stdout, stderr=stderr,
                preexec_fn=preexec_fn, close_fds=close_fds, shell=shell,
                cwd=cwd, env=env, universal_newlines=universal_newlines,
                startupinfo=startupinfo, creationflags=creationflags )

try:
    from subprocess import Popen, PIPE
except ImportError:
    Popen = PopenConstr
    PIPE = -1

class SvnBackupOutput:

    def __init__( self, absfilename ):
        self.__filename = absfilename

    def open( self ):
        pass

    def write( self, data ):
        pass

    def close( self ):
        pass

    def get_filename( self ):
        return self.__filename


class SvnBackupOutputPlain(SvnBackupOutput):

    def __init__( self, absfilename ):
        SvnBackupOutput.__init__( self, absfilename )

    def open( self ):
        self.__ofd = open( self.get_filename(), "wb" )

    def write( self, data ):
        self.__ofd.write( data )

    def close( self ):
        self.__ofd.close()


class SvnBackupOutputGzip(SvnBackupOutput):

    def __init__( self, absfilename ):
        SvnBackupOutput.__init__( self, absfilename + ".gz" )

    def open( self ):
        self.__compressor = gzip.GzipFile( filename=self.get_filename(),
                mode="wb" )

    def write( self, data ):
        self.__compressor.write( data )

    def close( self ):
        self.__compressor.flush()
        self.__compressor.close()


class SvnBackupOutputBzip2(SvnBackupOutput):

    def __init__( self, absfilename ):
        SvnBackupOutput.__init__( self, absfilename + ".bz2" )

    def open( self ):
        self.__compressor = bz2.BZ2Compressor()
        self.__ofd = open( self.get_filename(), "wb" )

    def write( self, data ):
        self.__ofd.write( self.__compressor.compress( data ) )

    def close( self ):
        self.__ofd.write( self.__compressor.flush() )
        self.__ofd.close()


class SvnBackupException( Exception ):

    def __init__( self, errortext ):
        self.errortext = errortext

    def __str__( self ):
        return self.errortext

class SvnBackup:

    def __init__( self, options, args ):
        # need 3 args: progname, reposname, dumpdir
        if len(args) != 3:
            if len(args) < 3:
                raise SvnBackupException, \
                    "too few arguments, specify repospath and dumpdir."
            else:
                raise SvnBackupException, \
                    "too many arguments, specify repospath and dumpdir only."
        self.__repospath = args[1]
        self.__dumpdir = args[2]
        # check repospath
        rpathparts = os.path.split( self.__repospath )
        if len( rpathparts[1] ) == 0:
            # repospath without trailing slash
            self.__repospath = rpathparts[0]
        if not os.path.exists( self.__repospath ):
            raise SvnBackupException, \
                "repos '%s' does not exist." % self.__repospath
        if not os.path.isdir( self.__repospath ):
            raise SvnBackupException, \
                "repos '%s' is not a directory." % self.__repospath
        for subdir in [ "db", "conf", "hooks" ]:
            dir = os.path.join( self.__repospath, "db" )
            if not os.path.isdir( dir ):
                raise SvnBackupException, \
                    "repos '%s' is not a repository." % self.__repospath
        rpathparts = os.path.split( self.__repospath )
        self.__reposname = rpathparts[1]
        if self.__reposname in [ "", ".", ".." ]:
            raise SvnBackupException, \
                "couldn't extract repos name from '%s'." % self.__repospath
        # check dumpdir
        if not os.path.exists( self.__dumpdir ):
            raise SvnBackupException, \
                "dumpdir '%s' does not exist." % self.__dumpdir
        elif not os.path.isdir( self.__dumpdir ):
            raise SvnBackupException, \
                "dumpdir '%s' is not a directory." % self.__dumpdir
        # set options
        self.__rev_nr = options.rev
        self.__count = options.cnt
        self.__quiet = options.quiet
        self.__deltas = options.deltas
        self.__zip = options.zip
        self.__overwrite = False
        self.__overwrite_all = False
        if options.overwrite > 0:
            self.__overwrite = True
        if options.overwrite > 1:
            self.__overwrite_all = True
        self.__transfer = None
        if options.transfer != None:
            self.__transfer = options.transfer.split( ":" )
            if len( self.__transfer ) != 5:
                if len( self.__transfer ) < 5:
                    raise SvnBackupException, \
                        "too few fields for transfer '%s'." % self.__transfer
                else:
                    raise SvnBackupException, \
                        "too many fields for transfer '%s'." % self.__transfer
            if self.__transfer[0] not in [ "ftp", "smb" ]:
                raise SvnBackupException, \
                    "unknown transfer method '%s'." % self.__transfer[0]

    def set_nonblock( self, fileobj ):
        fd = fileobj.fileno()
        n = fcntl.fcntl( fd, fcntl.F_GETFL )
        fcntl.fcntl( fd, fcntl.F_SETFL, n|os.O_NONBLOCK )

    def exec_cmd( self, cmd, output=None, printerr=False ):
        try:
            proc = Popen( cmd, stdout=PIPE, stderr=PIPE, shell=False )
        except:
            return ( 256, "", "Popen failed (%s ...):\n  %s" % ( cmd[0],
                    str(sys.exc_info()[1]) ) )
        stdout = proc.stdout
        stderr = proc.stderr
        self.set_nonblock( stdout )
        self.set_nonblock( stderr )
        readfds = [ stdout, stderr ]
        selres = select.select( readfds, [], [] )
        bufout = ""
        buferr = ""
        while len( selres[0] ) > 0:
            for fd in selres[0]:
                buf = fd.read( 16384 )
                if len( buf ) == 0:
                    readfds.remove( fd )
                elif fd == stdout:
                    if output:
                        output.write( buf )
                    else:
                        bufout += buf
                else:
                    if printerr:
                        print buf,
                    else:
                        buferr += buf
            if len( readfds ) == 0:
                break
            selres = select.select( readfds, [], [] )
        rc = proc.wait()
        if printerr:
            print ""
        return ( rc, bufout, buferr )

    def get_head_rev( self ):
        cmd = [ "svnlook", "youngest", self.__repospath ]
        r = self.exec_cmd( cmd )
        if r[0] == 0 and len( r[2] ) == 0:
            return int( r[1].strip() )
        else:
            print r[2]
        return -1

    def transfer_ftp( self, absfilename, filename ):
        rc = False
        try:
            host = self.__transfer[1]
            user = self.__transfer[2]
            passwd = self.__transfer[3]
            destdir = self.__transfer[4].replace( "%r", self.__reposname )
            ftp = FTP( host, user, passwd )
            ftp.cwd( destdir )
            ifd = open( absfilename, "rb" )
            ftp.storbinary( "STOR %s" % filename, ifd )
            ftp.quit()
            rc = len( ifd.read(1) ) == 0
            ifd.close()
        except Exception, e:
            raise SvnBackupException, \
                "ftp transfer failed:\n  file:  '%s'\n  error: %s" % \
                    ( absfilename, str(e) )
        return rc

    def transfer_smb( self, absfilename, filename ):
        share = self.__transfer[1]
        user = self.__transfer[2]
        passwd = self.__transfer[3]
        if passwd == "":
            passwd = "-N"
        destdir = self.__transfer[4].replace( "%r", self.__reposname )
        cmd = ( "smbclient", share, "-U", user, passwd, "-D", destdir,
                "-c", "put %s %s" % ( absfilename, filename ) )
        r = self.exec_cmd( cmd )
        rc = r[0] == 0
        if not rc:
            print r[2]
        return rc

    def transfer( self, absfilename, filename ):
        if self.__transfer == None:
            return
        elif self.__transfer[0] == "ftp":
            self.transfer_ftp( absfilename, filename )
        elif self.__transfer[0] == "smb":
            self.transfer_smb( absfilename, filename )
        else:
            print "unknown transfer method '%s'." % self.__transfer[0]

    def create_dump( self, checkonly, overwrite, fromrev, torev=None ):
        revparam = "%d" % fromrev
        r = "%06d" % fromrev
        if torev != None:
            revparam += ":%d" % torev
            r += "-%06d" % torev
        filename = "%s.%s.svndmp" % ( self.__reposname, r )
        absfilename = "%s/%s" % ( self.__dumpdir, filename )
        output = None
        if self.__zip:
            if self.__zip == "gzip":
                output = SvnBackupOutputGzip( absfilename )
            else:
                output = SvnBackupOutputBzip2( absfilename )
        else:
            output = SvnBackupOutputPlain( absfilename )
        realfilename = output.get_filename()
        if checkonly:
            return os.path.exists( realfilename )
        elif os.path.exists( realfilename ):
            if overwrite:
                print "overwriting " + realfilename
            else:
                print "%s already exists." % realfilename
                return True
        else:
            print "writing " + realfilename
        cmd = [ "svnadmin", "dump",
                "--incremental", "-r", revparam, self.__repospath ]
        if self.__quiet:
            cmd[2:2] = [ "-q" ]
        if self.__deltas:
            cmd[2:2] = [ "--deltas" ]
        output.open()
        r = self.exec_cmd( cmd, output, True )
        output.close()
        rc = r[0] == 0
        if rc:
            self.transfer( output.get_filename(), filename )
        return rc

    def export_single_rev( self ):
        return self.create_dump( False, self.__overwrite, self.__rev_nr )

    def export( self ):
        headrev = self.get_head_rev()
        if headrev == -1:
            return False
        if self.__count is None:
            return self.create_dump( False, self.__overwrite, 0, headrev )
        baserev = headrev - ( headrev % self.__count )
        rc = True
        cnt = self.__count
        fromrev = baserev - cnt
        torev = baserev - 1
        while fromrev >= 0 and rc:
            if self.__overwrite_all or \
                not self.create_dump( True, False, fromrev, torev ):
                rc = self.create_dump( False, self.__overwrite_all,
                            fromrev, torev )
                fromrev -= cnt
                torev -= cnt
            else:
                fromrev = -1
        if rc:
            rc = self.create_dump( False, self.__overwrite, baserev, headrev )
        return rc

    def execute( self ):
        if self.__rev_nr != None:
            return self.export_single_rev()
        else:
            return self.export()


if __name__ == "__main__":
    usage = "usage: svnbackup.py [options] repospath dumpdir"
    parser = OptionParser( usage=usage, version="%prog "+__version )
    if have_bz2:
        parser.add_option( "-b",
                       action="store_const", const="bzip2",
                       dest="zip", default=None,
                       help="compress the dump using bzip2." )
    parser.add_option( "--deltas",
                       action="store_true",
                       dest="deltas", default=False,
                       help="pass --deltas to svnadmin dump." )
    parser.add_option( "-c",
                       action="store", type="int",
                       dest="cnt", default=None,
                       help="count of revisions per dumpfile." )
    parser.add_option( "-o",
                       action="store_const", const=1,
                       dest="overwrite", default=0,
                       help="overwrite files." )
    parser.add_option( "-O",
                       action="store_const", const=2,
                       dest="overwrite", default=0,
                       help="overwrite all files." )
    parser.add_option( "-q",
                       action="store_true",
                       dest="quiet", default=False,
                       help="quiet." )
    parser.add_option( "-r",
                       action="store", type="int",
                       dest="rev", default=None,
                       help="revision number for single rev dump." )
    parser.add_option( "-t",
                       action="store", type="string",
                       dest="transfer", default=None,
                       help="transfer dumps to another machine "+
                            "(s.a. --help-transfer)." )
    parser.add_option( "-z",
                       action="store_const", const="gzip",
                       dest="zip",
                       help="compress the dump using gzip." )
    parser.add_option( "--help-transfer",
                       action="store_true",
                       dest="help_transfer", default=False,
                       help="shows detailed help for the transfer option." )
    (options, args) = parser.parse_args( sys.argv )
    if options.help_transfer:
        print "Transfer help:"
        print ""
        print "  FTP:"
        print "    -t ftp:<host>:<user>:<password>:<dest-path>"
        print ""
        print "  SMB (using smbclient):"
        print "    -t smb:<share>:<user>:<password>:<dest-path>"
        print ""
        sys.exit( 0 )
    rc = False
    try:
        backup = SvnBackup( options, args )
        rc = backup.execute()
    except SvnBackupException, e:
        print "svn-backup-dumps.py:", e
    if rc:
        print "Everything OK."
        sys.exit( 0 )
    else:
        print "An error occured!"
        sys.exit( 1 )

# vim:et:ts=4:sw=4