13
15
# You should have received a copy of the GNU General Public License
14
16
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
19
"""A collection of function for handling URL operations."""
22
from posixpath import split as _posix_split, normpath as _posix_normpath
23
from bzrlib.lazy_import import lazy_import
24
lazy_import(globals(), """
25
from posixpath import split as _posix_split, normpath as _posix_normpath
27
import bzrlib.errors as errors
36
31
def basename(url, exclude_trailing_slash=True):
66
61
relpath = relpath.encode('utf-8')
67
62
# After quoting and encoding, the path should be perfectly
68
63
# safe as a plain ASCII string, str() just enforces this
69
return str(urllib.quote(relpath, safe='/~'))
64
return str(urllib.quote(relpath))
72
67
def file_relpath(base, path):
73
68
"""Compute just the relative sub-portion of a url
75
70
This assumes that both paths are already fully specified file:// URLs.
77
if len(base) < MIN_ABS_FILEURL_LENGTH:
78
raise ValueError('Length of base (%r) must equal or'
79
' exceed the platform minimum url length (which is %d)' %
80
(base, MIN_ABS_FILEURL_LENGTH))
72
assert len(base) >= MIN_ABS_FILEURL_LENGTH, ('Length of base must be equal or'
73
' exceed the platform minimum url length (which is %d)' %
74
MIN_ABS_FILEURL_LENGTH)
81
76
base = local_path_from_url(base)
82
77
path = local_path_from_url(path)
83
return escape(osutils.relpath(base, path))
78
return escape(bzrlib.osutils.relpath(base, path))
86
81
def _find_scheme_and_separator(url):
132
125
path = base.split('/')
134
if scheme is not None and len(path) >= 1:
136
# the path should be represented as an abs path.
137
# we know this must be absolute because of the presence of a URL scheme.
139
path = [''] + path[1:]
141
# create an empty host, but dont alter the path - this might be a
142
# relative url fragment.
147
match = _url_scheme_re.match(arg)
128
m = _url_scheme_re.match(arg)
150
scheme = match.group('scheme')
151
# this skips .. normalisation, making http://host/../../..
153
path = match.group('path').split('/')
154
# set the host and path according to new absolute URL, discarding
155
# any previous values.
156
# XXX: duplicates mess from earlier in this function. This URL
157
# manipulation code needs some cleaning up.
158
if scheme is not None and len(path) >= 1:
161
# url scheme implies absolute path.
164
# no url scheme we take the path as is.
131
scheme = m.group('scheme')
132
path = m.group('path').split('/')
167
path = '/'.join(path)
168
path = joinpath(path, arg)
169
path = path.split('/')
170
if remove_root and path[0:1] == ['']:
173
# Remove the leading slash from the path, so long as it isn't also the
174
# trailing slash, which we want to keep if present.
175
if path and path[0] == '' and len(path) > 1:
134
for chunk in arg.split('/'):
139
# Don't pop off the host portion
142
raise errors.InvalidURLJoin('Cannot go above root',
179
147
if scheme is None:
180
148
return '/'.join(path)
181
149
return scheme + '://' + '/'.join(path)
184
def joinpath(base, *args):
185
"""Join URL path segments to a URL path segment.
187
This is somewhat like osutils.joinpath, but intended for URLs.
189
XXX: this duplicates some normalisation logic, and also duplicates a lot of
190
path handling logic that already exists in some Transport implementations.
191
We really should try to have exactly one place in the code base responsible
192
for combining paths of URLs.
194
path = base.split('/')
195
if len(path) > 1 and path[-1] == '':
196
#If the path ends in a trailing /, remove it.
199
if arg.startswith('/'):
201
for chunk in arg.split('/'):
206
raise errors.InvalidURLJoin('Cannot go above root',
214
return '/'.join(path)
217
152
# jam 20060502 Sorted to 'l' because the final target is 'local_path_from_url'
218
153
def _posix_local_path_from_url(url):
219
154
"""Convert a url like file:///path/to/foo into /path/to/foo"""
229
164
This also handles transforming escaping unicode characters, etc.
231
# importing directly from posixpath allows us to test this
166
# importing directly from posixpath allows us to test this
232
167
# on non-posix platforms
233
168
return 'file://' + escape(_posix_normpath(
234
osutils._posix_abspath(path)))
169
bzrlib.osutils._posix_abspath(path)))
237
172
def _win32_local_path_from_url(url):
238
173
"""Convert a url like file:///C:/path/to/foo into C:/path/to/foo"""
239
if not url.startswith('file://'):
240
raise errors.InvalidURL(url, 'local urls must start with file:///, '
241
'UNC path urls must start with file://')
174
if not url.startswith('file:///'):
175
raise errors.InvalidURL(url, 'local urls must start with file:///')
242
176
# We strip off all 3 slashes
243
win32_url = url[len('file:'):]
244
# check for UNC path: //HOST/path
245
if not win32_url.startswith('///'):
246
if (win32_url[2] == '/'
247
or win32_url[3] in '|:'):
248
raise errors.InvalidURL(url, 'Win32 UNC path urls'
249
' have form file://HOST/path')
250
return unescape(win32_url)
252
# allow empty paths so we can serve all roots
253
if win32_url == '///':
256
# usual local path with drive letter
257
if (win32_url[3] not in ('abcdefghijklmnopqrstuvwxyz'
177
win32_url = url[len('file:///'):]
178
if (win32_url[0] not in ('abcdefghijklmnopqrstuvwxyz'
258
179
'ABCDEFGHIJKLMNOPQRSTUVWXYZ')
259
or win32_url[4] not in '|:'
260
or win32_url[5] != '/'):
180
or win32_url[1] not in '|:'
181
or win32_url[2] != '/'):
261
182
raise errors.InvalidURL(url, 'Win32 file urls start with'
262
183
' file:///x:/, where x is a valid drive letter')
263
return win32_url[3].upper() + u':' + unescape(win32_url[5:])
184
# Preferentially using .lower() because os.getcwd() returns
185
# paths with lowercase drive letters, and that helps
186
# bzrlib.osutils.relpath() work correctly
187
return win32_url[0].lower() + u':' + unescape(win32_url[2:])
266
190
def _win32_local_path_to_url(path):
269
193
This also handles transforming escaping unicode characters, etc.
271
# importing directly from ntpath allows us to test this
195
# importing directly from ntpath allows us to test this
272
196
# on non-win32 platform
273
197
# FIXME: It turns out that on nt, ntpath.abspath uses nt._getfullpathname
274
198
# which actually strips trailing space characters.
275
199
# The worst part is that under linux ntpath.abspath has different
276
200
# semantics, since 'nt' is not an available module.
280
win32_path = osutils._win32_abspath(path)
281
# check for UNC path \\HOST\path
282
if win32_path.startswith('//'):
283
return 'file:' + escape(win32_path)
284
return ('file:///' + str(win32_path[0].upper()) + ':' +
285
escape(win32_path[2:]))
201
win32_path = bzrlib.osutils._nt_normpath(
202
bzrlib.osutils._win32_abspath(path)).replace('\\', '/')
203
return 'file:///' + win32_path[0].lower() + ':' + escape(win32_path[2:])
288
206
local_path_to_url = _posix_local_path_to_url
300
218
_url_scheme_re = re.compile(r'^(?P<scheme>[^:/]{2,})://(?P<path>.*)$')
301
_url_hex_escapes_re = re.compile(r'(%[0-9a-fA-F]{2})')
304
def _unescape_safe_chars(matchobj):
305
"""re.sub callback to convert hex-escapes to plain characters (if safe).
307
e.g. '%7E' will be converted to '~'.
309
hex_digits = matchobj.group(0)[1:]
310
char = chr(int(hex_digits, 16))
311
if char in _url_dont_escape_characters:
314
return matchobj.group(0).upper()
317
221
def normalize_url(url):
318
222
"""Make sure that a path string is in fully normalized URL form.
320
This handles URLs which have unicode characters, spaces,
224
This handles URLs which have unicode characters, spaces,
321
225
special characters, etc.
323
227
It has two basic modes of operation, depending on whether the
336
240
m = _url_scheme_re.match(url)
338
242
return local_path_to_url(url)
339
scheme = m.group('scheme')
340
path = m.group('path')
341
243
if not isinstance(url, unicode):
343
245
if c not in _url_safe_characters:
344
246
raise errors.InvalidURL(url, 'URLs can only contain specific'
345
247
' safe characters (not %r)' % c)
346
path = _url_hex_escapes_re.sub(_unescape_safe_chars, path)
347
return str(scheme + '://' + ''.join(path))
349
249
# We have a unicode (hybrid) url
350
path_chars = list(path)
250
scheme = m.group('scheme')
251
path = list(m.group('path'))
352
for i in xrange(len(path_chars)):
353
if path_chars[i] not in _url_safe_characters:
354
chars = path_chars[i].encode('utf-8')
355
path_chars[i] = ''.join(
356
['%%%02X' % ord(c) for c in path_chars[i].encode('utf-8')])
357
path = ''.join(path_chars)
358
path = _url_hex_escapes_re.sub(_unescape_safe_chars, path)
359
return str(scheme + '://' + path)
253
for i in xrange(len(path)):
254
if path[i] not in _url_safe_characters:
255
chars = path[i].encode('utf-8')
256
path[i] = ''.join(['%%%02X' % ord(c) for c in path[i].encode('utf-8')])
257
return scheme + '://' + ''.join(path)
362
260
def relative_url(base, other):
540
433
# These are characters that if escaped, should stay that way
541
434
_no_decode_chars = ';/?:@&=+$,#'
542
435
_no_decode_ords = [ord(c) for c in _no_decode_chars]
543
_no_decode_hex = (['%02x' % o for o in _no_decode_ords]
436
_no_decode_hex = (['%02x' % o for o in _no_decode_ords]
544
437
+ ['%02X' % o for o in _no_decode_ords])
545
438
_hex_display_map = dict(([('%02x' % o, chr(o)) for o in range(256)]
546
439
+ [('%02X' % o, chr(o)) for o in range(256)]))
547
440
#These entries get mapped to themselves
548
441
_hex_display_map.update((hex,'%'+hex) for hex in _no_decode_hex)
550
# These characters shouldn't be percent-encoded, and it's always safe to
551
# unencode them if they are.
552
_url_dont_escape_characters = set(
553
"abcdefghijklmnopqrstuvwxyz" # Lowercase alpha
554
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" # Uppercase alpha
555
"0123456789" # Numbers
556
"-._~" # Unreserved characters
559
443
# These characters should not be escaped
560
_url_safe_characters = set(
561
"abcdefghijklmnopqrstuvwxyz" # Lowercase alpha
562
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" # Uppercase alpha
563
"0123456789" # Numbers
564
"_.-!~*'()" # Unreserved characters
565
"/;?:@&=+$," # Reserved characters
566
"%#" # Extra reserved characters
444
_url_safe_characters = set('abcdefghijklmnopqrstuvwxyz'
445
'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
569
450
def unescape_for_display(url, encoding):
570
451
"""Decode what you can for a URL, so that we get a nice looking path.
572
453
This will turn file:// urls into local paths, and try to decode
573
454
any portions of a http:// style url that it can.
575
Any sections of the URL which can't be represented in the encoding or
456
Any sections of the URL which can't be represented in the encoding or
576
457
need to stay as escapes are left alone.
578
459
:param url: A 7-bit ASCII URL
579
460
:param encoding: The final output encoding
581
:return: A unicode string which can be safely encoded into the
462
:return: A unicode string which can be safely encoded into the
582
463
specified encoding.
585
raise ValueError('you cannot specify None for the display encoding')
465
assert encoding is not None, 'you cannot specify None for the display encoding.'
586
466
if url.startswith('file://'):
588
468
path = local_path_from_url(url)
622
502
# Otherwise take the url decoded one
624
504
return u'/'.join(res)
627
def derive_to_location(from_location):
628
"""Derive a TO_LOCATION given a FROM_LOCATION.
630
The normal case is a FROM_LOCATION of http://foo/bar => bar.
631
The Right Thing for some logical destinations may differ though
632
because no / may be present at all. In that case, the result is
633
the full name without the scheme indicator, e.g. lp:foo-bar => foo-bar.
634
This latter case also applies when a Windows drive
635
is used without a path, e.g. c:foo-bar => foo-bar.
636
If no /, path separator or : is found, the from_location is returned.
638
if from_location.find("/") >= 0 or from_location.find(os.sep) >= 0:
639
return os.path.basename(from_location.rstrip("/\\"))
641
sep = from_location.find(":")
643
return from_location[sep+1:]
648
def _is_absolute(url):
649
return (osutils.pathjoin('/foo', url) == url)
652
def rebase_url(url, old_base, new_base):
653
"""Convert a relative path from an old base URL to a new base URL.
655
The result will be a relative path.
656
Absolute paths and full URLs are returned unaltered.
658
scheme, separator = _find_scheme_and_separator(url)
659
if scheme is not None:
661
if _is_absolute(url):
663
old_parsed = urlparse.urlparse(old_base)
664
new_parsed = urlparse.urlparse(new_base)
665
if (old_parsed[:2]) != (new_parsed[:2]):
666
raise errors.InvalidRebaseURLs(old_base, new_base)
667
return determine_relative_path(new_parsed[2],
668
join(old_parsed[2], url))
671
def determine_relative_path(from_path, to_path):
672
"""Determine a relative path from from_path to to_path."""
673
from_segments = osutils.splitpath(from_path)
674
to_segments = osutils.splitpath(to_path)
676
for count, (from_element, to_element) in enumerate(zip(from_segments,
678
if from_element != to_element:
682
unique_from = from_segments[count:]
683
unique_to = to_segments[count:]
684
segments = (['..'] * len(unique_from) + unique_to)
685
if len(segments) == 0:
687
return osutils.pathjoin(*segments)
692
"""Extract the server address, the credentials and the path from the url.
694
user, password, host and path should be quoted if they contain reserved
697
:param url: an quoted url
699
:return: (scheme, user, password, host, port, path) tuple, all fields
702
if isinstance(url, unicode):
703
raise errors.InvalidURL('should be ascii:\n%r' % url)
704
url = url.encode('utf-8')
705
(scheme, netloc, path, params,
706
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
707
user = password = host = port = None
709
user, host = netloc.rsplit('@', 1)
711
user, password = user.split(':', 1)
712
password = urllib.unquote(password)
713
user = urllib.unquote(user)
717
if ':' in host and not (host[0] == '[' and host[-1] == ']'): #there *is* port
718
host, port = host.rsplit(':',1)
722
raise errors.InvalidURL('invalid port number %s in url:\n%s' %
724
if host != "" and host[0] == '[' and host[-1] == ']': #IPv6
727
host = urllib.unquote(host)
728
path = urllib.unquote(path)
730
return (scheme, user, password, host, port, path)
added
removed
bzrlib/urlutils.py
