~bzr-pqm/bzr/bzr.dev

# Copyright (C) 2006 Canonical Ltd

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

"""RemoteTransport client for the smart-server.

This module shouldn't be accessed directly.  The classes defined here should be

imported from bzrlib.smart.

"""

__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']

from cStringIO import StringIO

import urllib

import urlparse

from bzrlib import (

    errors,

    transport,

    urlutils,

    )

from bzrlib.smart import client, medium, protocol

# must do this otherwise urllib can't parse the urls properly :(

for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:

    transport.register_urlparse_netloc_protocol(scheme)

del scheme

# Port 4155 is the default port for bzr://, registered with IANA.

BZR_DEFAULT_INTERFACE = '0.0.0.0'

BZR_DEFAULT_PORT = 4155

class _SmartStat(object):

    def __init__(self, size, mode):

        self.st_size = size

        self.st_mode = mode

class RemoteTransport(transport.Transport):

    """Connection to a smart server.

    The connection holds references to the medium that can be used to send

    requests to the server.

    The connection has a notion of the current directory to which it's

    connected; this is incorporated in filenames passed to the server.

    This supports some higher-level RPC operations and can also be treated 

    like a Transport to do file-like operations.

    The connection can be made over a tcp socket, an ssh pipe or a series of

    http requests.  There are concrete subclasses for each type:

    RemoteTCPTransport, etc.

    """

    # IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding

    # responsibilities: Put those on SmartClient or similar. This is vital for

    # the ability to support multiple versions of the smart protocol over time:

    # RemoteTransport is an adapter from the Transport object model to the 

    # SmartClient model, not an encoder.

    def __init__(self, url, clone_from=None, medium=None, _client=None):

        """Constructor.

        :param clone_from: Another RemoteTransport instance that this one is

            being cloned from.  Attributes such as credentials and the medium

            will be reused.

        :param medium: The medium to use for this RemoteTransport. This must be

            supplied if clone_from is None.

        :param _client: Override the _SmartClient used by this transport.  This

            should only be used for testing purposes; normally this is

            determined from the medium.

        """

        ### Technically super() here is faulty because Transport's __init__

        ### fails to take 2 parameters, and if super were to choose a silly

        ### initialisation order things would blow up. 

        if not url.endswith('/'):

            url += '/'

        super(RemoteTransport, self).__init__(url)

        self._scheme, self._username, self._password, self._host, self._port, self._path = \

                transport.split_url(url)

        if clone_from is None:

            self._medium = medium

        else:

            # credentials may be stripped from the base in some circumstances

            # as yet to be clearly defined or documented, so copy them.

            self._username = clone_from._username

            # reuse same connection

            self._medium = clone_from._medium

        assert self._medium is not None

        if _client is None:

            self._client = client._SmartClient(self._medium)

        else:

            self._client = _client

    def abspath(self, relpath):

        """Return the full url to the given relative path.

        @param relpath: the relative path or path components

        @type relpath: str or list

        """

        return self._unparse_url(self._remote_path(relpath))

    def clone(self, relative_url):

        """Make a new RemoteTransport related to me, sharing the same connection.

        This essentially opens a handle on a different remote directory.

        """

        if relative_url is None:

            return RemoteTransport(self.base, self)

        else:

            return RemoteTransport(self.abspath(relative_url), self)

    def is_readonly(self):

        """Smart server transport can do read/write file operations."""

        resp = self._call2('Transport.is_readonly')

        if resp == ('yes', ):

            return True

        elif resp == ('no', ):

            return False

        elif (resp == ('error', "Generic bzr smart protocol error: "

                                "bad request 'Transport.is_readonly'") or

              resp == ('error', "Generic bzr smart protocol error: "

                                "bad request u'Transport.is_readonly'")):

            # XXX: nasty hack: servers before 0.16 don't have a

            # 'Transport.is_readonly' verb, so we do what clients before 0.16

            # did: assume False.

            return False

        else:

            self._translate_error(resp)

        raise errors.UnexpectedSmartServerResponse(resp)

    def get_smart_client(self):

        return self._medium

    def get_smart_medium(self):

        return self._medium

    def _unparse_url(self, path):

        """Return URL for a path.

        :see: SFTPUrlHandling._unparse_url

        """

        # TODO: Eventually it should be possible to unify this with

        # SFTPUrlHandling._unparse_url?

        if path == '':

            path = '/'

        path = urllib.quote(path)

        netloc = urllib.quote(self._host)

        if self._username is not None:

            netloc = '%s@%s' % (urllib.quote(self._username), netloc)

        if self._port is not None:

            netloc = '%s:%d' % (netloc, self._port)

        return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))

    def _remote_path(self, relpath):

        """Returns the Unicode version of the absolute path for relpath."""

        return self._combine_paths(self._path, relpath)

    def _call(self, method, *args):

        resp = self._call2(method, *args)

        self._translate_error(resp)

    def _call2(self, method, *args):

        """Call a method on the remote server."""

        return self._client.call(method, *args)

    def _call_with_body_bytes(self, method, args, body):

        """Call a method on the remote server with body bytes."""

        return self._client.call_with_body_bytes(method, args, body)

    def has(self, relpath):

        """Indicate whether a remote file of the given name exists or not.

        :see: Transport.has()

        """

        resp = self._call2('has', self._remote_path(relpath))

        if resp == ('yes', ):

            return True

        elif resp == ('no', ):

            return False

        else:

            self._translate_error(resp)

    def get(self, relpath):

        """Return file-like object reading the contents of a remote file.

        :see: Transport.get_bytes()/get_file()

        """

        return StringIO(self.get_bytes(relpath))

    def get_bytes(self, relpath):

        remote = self._remote_path(relpath)

        request = self._medium.get_request()

        smart_protocol = protocol.SmartClientRequestProtocolOne(request)

        smart_protocol.call('get', remote)

        resp = smart_protocol.read_response_tuple(True)

        if resp != ('ok', ):

            smart_protocol.cancel_read_body()

            self._translate_error(resp, relpath)

        return smart_protocol.read_body_bytes()

    def _serialise_optional_mode(self, mode):

        if mode is None:

            return ''

        else:

            return '%d' % mode

    def mkdir(self, relpath, mode=None):

        resp = self._call2('mkdir', self._remote_path(relpath),

            self._serialise_optional_mode(mode))

        self._translate_error(resp)

    def put_bytes(self, relpath, upload_contents, mode=None):

        # FIXME: upload_file is probably not safe for non-ascii characters -

        # should probably just pass all parameters as length-delimited

        # strings?

        if type(upload_contents) is unicode:

            # Although not strictly correct, we raise UnicodeEncodeError to be

            # compatible with other transports.

            raise UnicodeEncodeError(

                'undefined', upload_contents, 0, 1,

                'put_bytes must be given bytes, not unicode.')

        resp = self._call_with_body_bytes('put',

            (self._remote_path(relpath), self._serialise_optional_mode(mode)),

            upload_contents)

        self._translate_error(resp)

    def put_bytes_non_atomic(self, relpath, bytes, mode=None,

                             create_parent_dir=False,

                             dir_mode=None):

        """See Transport.put_bytes_non_atomic."""

        # FIXME: no encoding in the transport!

        create_parent_str = 'F'

        if create_parent_dir:

            create_parent_str = 'T'

        resp = self._call_with_body_bytes(

            'put_non_atomic',

            (self._remote_path(relpath), self._serialise_optional_mode(mode),

             create_parent_str, self._serialise_optional_mode(dir_mode)),

            bytes)

        self._translate_error(resp)

    def put_file(self, relpath, upload_file, mode=None):

        # its not ideal to seek back, but currently put_non_atomic_file depends

        # on transports not reading before failing - which is a faulty

        # assumption I think - RBC 20060915

        pos = upload_file.tell()

        try:

            return self.put_bytes(relpath, upload_file.read(), mode)

        except:

            upload_file.seek(pos)

            raise

    def put_file_non_atomic(self, relpath, f, mode=None,

                            create_parent_dir=False,

                            dir_mode=None):

        return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,

                                         create_parent_dir=create_parent_dir,

                                         dir_mode=dir_mode)

    def append_file(self, relpath, from_file, mode=None):

        return self.append_bytes(relpath, from_file.read(), mode)

    def append_bytes(self, relpath, bytes, mode=None):

        resp = self._call_with_body_bytes(

            'append',

            (self._remote_path(relpath), self._serialise_optional_mode(mode)),

            bytes)

        if resp[0] == 'appended':

            return int(resp[1])

        self._translate_error(resp)

    def delete(self, relpath):

        resp = self._call2('delete', self._remote_path(relpath))

        self._translate_error(resp)

    def readv(self, relpath, offsets):

        if not offsets:

            return

        offsets = list(offsets)

        sorted_offsets = sorted(offsets)

        # turn the list of offsets into a stack

        offset_stack = iter(offsets)

        cur_offset_and_size = offset_stack.next()

        coalesced = list(self._coalesce_offsets(sorted_offsets,

                               limit=self._max_readv_combine,

                               fudge_factor=self._bytes_to_read_before_seek))

        request = self._medium.get_request()

        smart_protocol = protocol.SmartClientRequestProtocolOne(request)

        smart_protocol.call_with_body_readv_array(

            ('readv', self._remote_path(relpath)),

            [(c.start, c.length) for c in coalesced])

        resp = smart_protocol.read_response_tuple(True)

        if resp[0] != 'readv':

            # This should raise an exception

            smart_protocol.cancel_read_body()

            self._translate_error(resp)

            return

        # FIXME: this should know how many bytes are needed, for clarity.

        data = smart_protocol.read_body_bytes()

        # Cache the results, but only until they have been fulfilled

        data_map = {}

        for c_offset in coalesced:

            if len(data) < c_offset.length:

                raise errors.ShortReadvError(relpath, c_offset.start,

                            c_offset.length, actual=len(data))

            for suboffset, subsize in c_offset.ranges:

                key = (c_offset.start+suboffset, subsize)

                data_map[key] = data[suboffset:suboffset+subsize]

            data = data[c_offset.length:]

            # Now that we've read some data, see if we can yield anything back

            while cur_offset_and_size in data_map:

                this_data = data_map.pop(cur_offset_and_size)

                yield cur_offset_and_size[0], this_data

                cur_offset_and_size = offset_stack.next()

    def rename(self, rel_from, rel_to):

        self._call('rename',

                   self._remote_path(rel_from),

                   self._remote_path(rel_to))

    def move(self, rel_from, rel_to):

        self._call('move',

                   self._remote_path(rel_from),

                   self._remote_path(rel_to))

    def rmdir(self, relpath):

        resp = self._call('rmdir', self._remote_path(relpath))

    def _translate_error(self, resp, orig_path=None):

        """Raise an exception from a response"""

        if resp is None:

            what = None

        else:

            what = resp[0]

        if what == 'ok':

            return

        elif what == 'NoSuchFile':

            if orig_path is not None:

                error_path = orig_path

            else:

                error_path = resp[1]

            raise errors.NoSuchFile(error_path)

        elif what == 'error':

            raise errors.SmartProtocolError(unicode(resp[1]))

        elif what == 'FileExists':

            raise errors.FileExists(resp[1])

        elif what == 'DirectoryNotEmpty':

            raise errors.DirectoryNotEmpty(resp[1])

        elif what == 'ShortReadvError':

            raise errors.ShortReadvError(resp[1], int(resp[2]),

                                         int(resp[3]), int(resp[4]))

        elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):

            encoding = str(resp[1]) # encoding must always be a string

            val = resp[2]

            start = int(resp[3])

            end = int(resp[4])

            reason = str(resp[5]) # reason must always be a string

            if val.startswith('u:'):

                val = val[2:].decode('utf-8')

            elif val.startswith('s:'):

                val = val[2:].decode('base64')

            if what == 'UnicodeDecodeError':

                raise UnicodeDecodeError(encoding, val, start, end, reason)

            elif what == 'UnicodeEncodeError':

                raise UnicodeEncodeError(encoding, val, start, end, reason)

        elif what == "ReadOnlyError":

            raise errors.TransportNotPossible('readonly transport')

        elif what == "ReadError":

            if orig_path is not None:

                error_path = orig_path

            else:

                error_path = resp[1]

            raise errors.ReadError(error_path)

        else:

            raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))

    def disconnect(self):

        self._medium.disconnect()

    def delete_tree(self, relpath):

        raise errors.TransportNotPossible('readonly transport')

    def stat(self, relpath):

        resp = self._call2('stat', self._remote_path(relpath))

        if resp[0] == 'stat':

            return _SmartStat(int(resp[1]), int(resp[2], 8))

        else:

            self._translate_error(resp)

    ## def lock_read(self, relpath):

    ##     """Lock the given file for shared (read) access.

    ##     :return: A lock object, which should be passed to Transport.unlock()

    ##     """

    ##     # The old RemoteBranch ignore lock for reading, so we will

    ##     # continue that tradition and return a bogus lock object.

    ##     class BogusLock(object):

    ##         def __init__(self, path):

    ##             self.path = path

    ##         def unlock(self):

    ##             pass

    ##     return BogusLock(relpath)

    def listable(self):

        return True

    def list_dir(self, relpath):

        resp = self._call2('list_dir', self._remote_path(relpath))

        if resp[0] == 'names':

            return [name.encode('ascii') for name in resp[1:]]

        else:

            self._translate_error(resp)

    def iter_files_recursive(self):

        resp = self._call2('iter_files_recursive', self._remote_path(''))

        if resp[0] == 'names':

            return resp[1:]

        else:

            self._translate_error(resp)

class RemoteTCPTransport(RemoteTransport):

    """Connection to smart server over plain tcp.

    This is essentially just a factory to get 'RemoteTransport(url,

        SmartTCPClientMedium).

    """

    def __init__(self, url):

        _scheme, _username, _password, _host, _port, _path = \

            transport.split_url(url)

        if _port is None:

            _port = BZR_DEFAULT_PORT

        else:

            try:

                _port = int(_port)

            except (ValueError, TypeError), e:

                raise errors.InvalidURL(

                    path=url, extra="invalid port %s" % _port)

        client_medium = medium.SmartTCPClientMedium(_host, _port)

        super(RemoteTCPTransport, self).__init__(url, medium=client_medium)

class RemoteSSHTransport(RemoteTransport):

    """Connection to smart server over SSH.

    This is essentially just a factory to get 'RemoteTransport(url,

        SmartSSHClientMedium).

    """

    def __init__(self, url):

        _scheme, _username, _password, _host, _port, _path = \

            transport.split_url(url)

        try:

            if _port is not None:

                _port = int(_port)

        except (ValueError, TypeError), e:

            raise errors.InvalidURL(path=url, extra="invalid port %s" % 

                _port)

        client_medium = medium.SmartSSHClientMedium(_host, _port,

                                                    _username, _password)

        super(RemoteSSHTransport, self).__init__(url, medium=client_medium)

class RemoteHTTPTransport(RemoteTransport):

    """Just a way to connect between a bzr+http:// url and http://.

    This connection operates slightly differently than the RemoteSSHTransport.

    It uses a plain http:// transport underneath, which defines what remote

    .bzr/smart URL we are connected to. From there, all paths that are sent are

    sent as relative paths, this way, the remote side can properly

    de-reference them, since it is likely doing rewrite rules to translate an

    HTTP path into a local path.

    """

    def __init__(self, url, http_transport=None):

        assert url.startswith('bzr+http://')

        if http_transport is None:

            http_url = url[len('bzr+'):]

            self._http_transport = transport.get_transport(http_url)

        else:

            self._http_transport = http_transport

        http_medium = self._http_transport.get_smart_medium()

        super(RemoteHTTPTransport, self).__init__(url, medium=http_medium)

    def _remote_path(self, relpath):

        """After connecting HTTP Transport only deals in relative URLs."""

        # Adjust the relpath based on which URL this smart transport is

        # connected to.

        base = urlutils.normalize_url(self._http_transport.base)

        url = urlutils.join(self.base[len('bzr+'):], relpath)

        url = urlutils.normalize_url(url)

        return urlutils.relative_url(base, url)

    def abspath(self, relpath):

        """Return the full url to the given relative path.

        :param relpath: the relative path or path components

        :type relpath: str or list

        """

        return self._unparse_url(self._combine_paths(self._path, relpath))

    def clone(self, relative_url):

        """Make a new RemoteHTTPTransport related to me.

        This is re-implemented rather than using the default

        RemoteTransport.clone() because we must be careful about the underlying

        http transport.

        Also, the cloned smart transport will POST to the same .bzr/smart

        location as this transport (although obviously the relative paths in the

        smart requests may be different).  This is so that the server doesn't

        have to handle .bzr/smart requests at arbitrary places inside .bzr

        directories, just at the initial URL the user uses.

        The exception is parent paths (i.e. relative_url of "..").

        """

        if relative_url:

            abs_url = self.abspath(relative_url)

        else:

            abs_url = self.base

        # We either use the exact same http_transport (for child locations), or

        # a clone of the underlying http_transport (for parent locations).  This

        # means we share the connection.

        norm_base = urlutils.normalize_url(self.base)

        norm_abs_url = urlutils.normalize_url(abs_url)

        normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)

        if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):

            http_transport = self._http_transport.clone(normalized_rel_url)

        else:

            http_transport = self._http_transport

        return RemoteHTTPTransport(abs_url, http_transport=http_transport)

def get_test_permutations():

    """Return (transport, server) permutations for testing."""

    ### We may need a little more test framework support to construct an

    ### appropriate RemoteTransport in the future.

    from bzrlib.smart import server

    return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]