1 | """Utilities to support packages."""
|
2 |
|
3 | # NOTE: This module must remain compatible with Python 2.3, as it is shared
|
4 | # by setuptools for distribution with Python 2.3 and up.
|
5 |
|
6 | import os
|
7 | import sys
|
8 | import imp
|
9 | import os.path
|
10 | from types import ModuleType
|
11 |
|
12 | __all__ = [
|
13 | 'get_importer', 'iter_importers', 'get_loader', 'find_loader',
|
14 | 'walk_packages', 'iter_modules', 'get_data',
|
15 | 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
|
16 | ]
|
17 |
|
18 | def read_code(stream):
|
19 | # This helper is needed in order for the PEP 302 emulation to
|
20 | # correctly handle compiled files
|
21 | import marshal
|
22 |
|
23 | magic = stream.read(4)
|
24 | if magic != imp.get_magic():
|
25 | return None
|
26 |
|
27 | stream.read(4) # Skip timestamp
|
28 | return marshal.load(stream)
|
29 |
|
30 |
|
31 | def simplegeneric(func):
|
32 | """Make a trivial single-dispatch generic function"""
|
33 | registry = {}
|
34 | def wrapper(*args, **kw):
|
35 | ob = args[0]
|
36 | try:
|
37 | cls = ob.__class__
|
38 | except AttributeError:
|
39 | cls = type(ob)
|
40 | try:
|
41 | mro = cls.__mro__
|
42 | except AttributeError:
|
43 | try:
|
44 | class cls(cls, object):
|
45 | pass
|
46 | mro = cls.__mro__[1:]
|
47 | except TypeError:
|
48 | mro = object, # must be an ExtensionClass or some such :(
|
49 | for t in mro:
|
50 | if t in registry:
|
51 | return registry[t](*args, **kw)
|
52 | else:
|
53 | return func(*args, **kw)
|
54 | try:
|
55 | wrapper.__name__ = func.__name__
|
56 | except (TypeError, AttributeError):
|
57 | pass # Python 2.3 doesn't allow functions to be renamed
|
58 |
|
59 | def register(typ, func=None):
|
60 | if func is None:
|
61 | return lambda f: register(typ, f)
|
62 | registry[typ] = func
|
63 | return func
|
64 |
|
65 | wrapper.__dict__ = func.__dict__
|
66 | wrapper.__doc__ = func.__doc__
|
67 | wrapper.register = register
|
68 | return wrapper
|
69 |
|
70 |
|
71 | def walk_packages(path=None, prefix='', onerror=None):
|
72 | """Yields (module_loader, name, ispkg) for all modules recursively
|
73 | on path, or, if path is None, all accessible modules.
|
74 |
|
75 | 'path' should be either None or a list of paths to look for
|
76 | modules in.
|
77 |
|
78 | 'prefix' is a string to output on the front of every module name
|
79 | on output.
|
80 |
|
81 | Note that this function must import all *packages* (NOT all
|
82 | modules!) on the given path, in order to access the __path__
|
83 | attribute to find submodules.
|
84 |
|
85 | 'onerror' is a function which gets called with one argument (the
|
86 | name of the package which was being imported) if any exception
|
87 | occurs while trying to import a package. If no onerror function is
|
88 | supplied, ImportErrors are caught and ignored, while all other
|
89 | exceptions are propagated, terminating the search.
|
90 |
|
91 | Examples:
|
92 |
|
93 | # list all modules python can access
|
94 | walk_packages()
|
95 |
|
96 | # list all submodules of ctypes
|
97 | walk_packages(ctypes.__path__, ctypes.__name__+'.')
|
98 | """
|
99 |
|
100 | def seen(p, m={}):
|
101 | if p in m:
|
102 | return True
|
103 | m[p] = True
|
104 |
|
105 | for importer, name, ispkg in iter_modules(path, prefix):
|
106 | yield importer, name, ispkg
|
107 |
|
108 | if ispkg:
|
109 | try:
|
110 | __import__(name)
|
111 | except ImportError:
|
112 | if onerror is not None:
|
113 | onerror(name)
|
114 | except Exception:
|
115 | if onerror is not None:
|
116 | onerror(name)
|
117 | else:
|
118 | raise
|
119 | else:
|
120 | path = getattr(sys.modules[name], '__path__', None) or []
|
121 |
|
122 | # don't traverse path items we've seen before
|
123 | path = [p for p in path if not seen(p)]
|
124 |
|
125 | for item in walk_packages(path, name+'.', onerror):
|
126 | yield item
|
127 |
|
128 |
|
129 | def iter_modules(path=None, prefix=''):
|
130 | """Yields (module_loader, name, ispkg) for all submodules on path,
|
131 | or, if path is None, all top-level modules on sys.path.
|
132 |
|
133 | 'path' should be either None or a list of paths to look for
|
134 | modules in.
|
135 |
|
136 | 'prefix' is a string to output on the front of every module name
|
137 | on output.
|
138 | """
|
139 |
|
140 | if path is None:
|
141 | importers = iter_importers()
|
142 | else:
|
143 | importers = map(get_importer, path)
|
144 |
|
145 | yielded = {}
|
146 | for i in importers:
|
147 | for name, ispkg in iter_importer_modules(i, prefix):
|
148 | if name not in yielded:
|
149 | yielded[name] = 1
|
150 | yield i, name, ispkg
|
151 |
|
152 |
|
153 | #@simplegeneric
|
154 | def iter_importer_modules(importer, prefix=''):
|
155 | if not hasattr(importer, 'iter_modules'):
|
156 | return []
|
157 | return importer.iter_modules(prefix)
|
158 |
|
159 | iter_importer_modules = simplegeneric(iter_importer_modules)
|
160 |
|
161 |
|
162 | class ImpImporter:
|
163 | """PEP 302 Importer that wraps Python's "classic" import algorithm
|
164 |
|
165 | ImpImporter(dirname) produces a PEP 302 importer that searches that
|
166 | directory. ImpImporter(None) produces a PEP 302 importer that searches
|
167 | the current sys.path, plus any modules that are frozen or built-in.
|
168 |
|
169 | Note that ImpImporter does not currently support being used by placement
|
170 | on sys.meta_path.
|
171 | """
|
172 |
|
173 | def __init__(self, path=None):
|
174 | self.path = path
|
175 |
|
176 | def find_module(self, fullname, path=None):
|
177 | # Note: we ignore 'path' argument since it is only used via meta_path
|
178 | subname = fullname.split(".")[-1]
|
179 | if subname != fullname and self.path is None:
|
180 | return None
|
181 | if self.path is None:
|
182 | path = None
|
183 | else:
|
184 | path = [os.path.realpath(self.path)]
|
185 | try:
|
186 | file, filename, etc = imp.find_module(subname, path)
|
187 | except ImportError:
|
188 | return None
|
189 | return ImpLoader(fullname, file, filename, etc)
|
190 |
|
191 | def iter_modules(self, prefix=''):
|
192 | if self.path is None or not os.path.isdir(self.path):
|
193 | return
|
194 |
|
195 | yielded = {}
|
196 | import inspect
|
197 | try:
|
198 | filenames = os.listdir(self.path)
|
199 | except OSError:
|
200 | # ignore unreadable directories like import does
|
201 | filenames = []
|
202 | filenames.sort() # handle packages before same-named modules
|
203 |
|
204 | for fn in filenames:
|
205 | modname = inspect.getmodulename(fn)
|
206 | if modname=='__init__' or modname in yielded:
|
207 | continue
|
208 |
|
209 | path = os.path.join(self.path, fn)
|
210 | ispkg = False
|
211 |
|
212 | if not modname and os.path.isdir(path) and '.' not in fn:
|
213 | modname = fn
|
214 | try:
|
215 | dircontents = os.listdir(path)
|
216 | except OSError:
|
217 | # ignore unreadable directories like import does
|
218 | dircontents = []
|
219 | for fn in dircontents:
|
220 | subname = inspect.getmodulename(fn)
|
221 | if subname=='__init__':
|
222 | ispkg = True
|
223 | break
|
224 | else:
|
225 | continue # not a package
|
226 |
|
227 | if modname and '.' not in modname:
|
228 | yielded[modname] = 1
|
229 | yield prefix + modname, ispkg
|
230 |
|
231 |
|
232 | class ImpLoader:
|
233 | """PEP 302 Loader that wraps Python's "classic" import algorithm
|
234 | """
|
235 | code = source = None
|
236 |
|
237 | def __init__(self, fullname, file, filename, etc):
|
238 | self.file = file
|
239 | self.filename = filename
|
240 | self.fullname = fullname
|
241 | self.etc = etc
|
242 |
|
243 | def load_module(self, fullname):
|
244 | self._reopen()
|
245 | try:
|
246 | mod = imp.load_module(fullname, self.file, self.filename, self.etc)
|
247 | finally:
|
248 | if self.file:
|
249 | self.file.close()
|
250 | # Note: we don't set __loader__ because we want the module to look
|
251 | # normal; i.e. this is just a wrapper for standard import machinery
|
252 | return mod
|
253 |
|
254 | def get_data(self, pathname):
|
255 | return open(pathname, "rb").read()
|
256 |
|
257 | def _reopen(self):
|
258 | if self.file and self.file.closed:
|
259 | mod_type = self.etc[2]
|
260 | if mod_type==imp.PY_SOURCE:
|
261 | self.file = open(self.filename, 'rU')
|
262 | elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
|
263 | self.file = open(self.filename, 'rb')
|
264 |
|
265 | def _fix_name(self, fullname):
|
266 | if fullname is None:
|
267 | fullname = self.fullname
|
268 | elif fullname != self.fullname:
|
269 | raise ImportError("Loader for module %s cannot handle "
|
270 | "module %s" % (self.fullname, fullname))
|
271 | return fullname
|
272 |
|
273 | def is_package(self, fullname):
|
274 | fullname = self._fix_name(fullname)
|
275 | return self.etc[2]==imp.PKG_DIRECTORY
|
276 |
|
277 | def get_code(self, fullname=None):
|
278 | fullname = self._fix_name(fullname)
|
279 | if self.code is None:
|
280 | mod_type = self.etc[2]
|
281 | if mod_type==imp.PY_SOURCE:
|
282 | source = self.get_source(fullname)
|
283 | self.code = compile(source, self.filename, 'exec')
|
284 | elif mod_type==imp.PY_COMPILED:
|
285 | self._reopen()
|
286 | try:
|
287 | self.code = read_code(self.file)
|
288 | finally:
|
289 | self.file.close()
|
290 | elif mod_type==imp.PKG_DIRECTORY:
|
291 | self.code = self._get_delegate().get_code()
|
292 | return self.code
|
293 |
|
294 | def get_source(self, fullname=None):
|
295 | fullname = self._fix_name(fullname)
|
296 | if self.source is None:
|
297 | mod_type = self.etc[2]
|
298 | if mod_type==imp.PY_SOURCE:
|
299 | self._reopen()
|
300 | try:
|
301 | self.source = self.file.read()
|
302 | finally:
|
303 | self.file.close()
|
304 | elif mod_type==imp.PY_COMPILED:
|
305 | if os.path.exists(self.filename[:-1]):
|
306 | f = open(self.filename[:-1], 'rU')
|
307 | self.source = f.read()
|
308 | f.close()
|
309 | elif mod_type==imp.PKG_DIRECTORY:
|
310 | self.source = self._get_delegate().get_source()
|
311 | return self.source
|
312 |
|
313 |
|
314 | def _get_delegate(self):
|
315 | return ImpImporter(self.filename).find_module('__init__')
|
316 |
|
317 | def get_filename(self, fullname=None):
|
318 | fullname = self._fix_name(fullname)
|
319 | mod_type = self.etc[2]
|
320 | if self.etc[2]==imp.PKG_DIRECTORY:
|
321 | return self._get_delegate().get_filename()
|
322 | elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
|
323 | return self.filename
|
324 | return None
|
325 |
|
326 |
|
327 | try:
|
328 | import zipimport
|
329 | from zipimport import zipimporter
|
330 |
|
331 | def iter_zipimport_modules(importer, prefix=''):
|
332 | dirlist = zipimport._zip_directory_cache[importer.archive].keys()
|
333 | dirlist.sort()
|
334 | _prefix = importer.prefix
|
335 | plen = len(_prefix)
|
336 | yielded = {}
|
337 | import inspect
|
338 | for fn in dirlist:
|
339 | if not fn.startswith(_prefix):
|
340 | continue
|
341 |
|
342 | fn = fn[plen:].split(os.sep)
|
343 |
|
344 | if len(fn)==2 and fn[1].startswith('__init__.py'):
|
345 | if fn[0] not in yielded:
|
346 | yielded[fn[0]] = 1
|
347 | yield fn[0], True
|
348 |
|
349 | if len(fn)!=1:
|
350 | continue
|
351 |
|
352 | modname = inspect.getmodulename(fn[0])
|
353 | if modname=='__init__':
|
354 | continue
|
355 |
|
356 | if modname and '.' not in modname and modname not in yielded:
|
357 | yielded[modname] = 1
|
358 | yield prefix + modname, False
|
359 |
|
360 | iter_importer_modules.register(zipimporter, iter_zipimport_modules)
|
361 |
|
362 | except ImportError:
|
363 | pass
|
364 |
|
365 |
|
366 | def get_importer(path_item):
|
367 | """Retrieve a PEP 302 importer for the given path item
|
368 |
|
369 | The returned importer is cached in sys.path_importer_cache
|
370 | if it was newly created by a path hook.
|
371 |
|
372 | If there is no importer, a wrapper around the basic import
|
373 | machinery is returned. This wrapper is never inserted into
|
374 | the importer cache (None is inserted instead).
|
375 |
|
376 | The cache (or part of it) can be cleared manually if a
|
377 | rescan of sys.path_hooks is necessary.
|
378 | """
|
379 | try:
|
380 | importer = sys.path_importer_cache[path_item]
|
381 | except KeyError:
|
382 | for path_hook in sys.path_hooks:
|
383 | try:
|
384 | importer = path_hook(path_item)
|
385 | break
|
386 | except ImportError:
|
387 | pass
|
388 | else:
|
389 | importer = None
|
390 | sys.path_importer_cache.setdefault(path_item, importer)
|
391 |
|
392 | if importer is None:
|
393 | try:
|
394 | importer = ImpImporter(path_item)
|
395 | except ImportError:
|
396 | importer = None
|
397 | return importer
|
398 |
|
399 |
|
400 | def iter_importers(fullname=""):
|
401 | """Yield PEP 302 importers for the given module name
|
402 |
|
403 | If fullname contains a '.', the importers will be for the package
|
404 | containing fullname, otherwise they will be importers for sys.meta_path,
|
405 | sys.path, and Python's "classic" import machinery, in that order. If
|
406 | the named module is in a package, that package is imported as a side
|
407 | effect of invoking this function.
|
408 |
|
409 | Non PEP 302 mechanisms (e.g. the Windows registry) used by the
|
410 | standard import machinery to find files in alternative locations
|
411 | are partially supported, but are searched AFTER sys.path. Normally,
|
412 | these locations are searched BEFORE sys.path, preventing sys.path
|
413 | entries from shadowing them.
|
414 |
|
415 | For this to cause a visible difference in behaviour, there must
|
416 | be a module or package name that is accessible via both sys.path
|
417 | and one of the non PEP 302 file system mechanisms. In this case,
|
418 | the emulation will find the former version, while the builtin
|
419 | import mechanism will find the latter.
|
420 |
|
421 | Items of the following types can be affected by this discrepancy:
|
422 | imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
|
423 | """
|
424 | if fullname.startswith('.'):
|
425 | raise ImportError("Relative module names not supported")
|
426 | if '.' in fullname:
|
427 | # Get the containing package's __path__
|
428 | pkg = '.'.join(fullname.split('.')[:-1])
|
429 | if pkg not in sys.modules:
|
430 | __import__(pkg)
|
431 | path = getattr(sys.modules[pkg], '__path__', None) or []
|
432 | else:
|
433 | for importer in sys.meta_path:
|
434 | yield importer
|
435 | path = sys.path
|
436 | for item in path:
|
437 | yield get_importer(item)
|
438 | if '.' not in fullname:
|
439 | yield ImpImporter()
|
440 |
|
441 | def get_loader(module_or_name):
|
442 | """Get a PEP 302 "loader" object for module_or_name
|
443 |
|
444 | If the module or package is accessible via the normal import
|
445 | mechanism, a wrapper around the relevant part of that machinery
|
446 | is returned. Returns None if the module cannot be found or imported.
|
447 | If the named module is not already imported, its containing package
|
448 | (if any) is imported, in order to establish the package __path__.
|
449 |
|
450 | This function uses iter_importers(), and is thus subject to the same
|
451 | limitations regarding platform-specific special import locations such
|
452 | as the Windows registry.
|
453 | """
|
454 | if module_or_name in sys.modules:
|
455 | module_or_name = sys.modules[module_or_name]
|
456 | if isinstance(module_or_name, ModuleType):
|
457 | module = module_or_name
|
458 | loader = getattr(module, '__loader__', None)
|
459 | if loader is not None:
|
460 | return loader
|
461 | fullname = module.__name__
|
462 | else:
|
463 | fullname = module_or_name
|
464 | return find_loader(fullname)
|
465 |
|
466 | def find_loader(fullname):
|
467 | """Find a PEP 302 "loader" object for fullname
|
468 |
|
469 | If fullname contains dots, path must be the containing package's __path__.
|
470 | Returns None if the module cannot be found or imported. This function uses
|
471 | iter_importers(), and is thus subject to the same limitations regarding
|
472 | platform-specific special import locations such as the Windows registry.
|
473 | """
|
474 | for importer in iter_importers(fullname):
|
475 | loader = importer.find_module(fullname)
|
476 | if loader is not None:
|
477 | return loader
|
478 |
|
479 | return None
|
480 |
|
481 |
|
482 | def extend_path(path, name):
|
483 | """Extend a package's path.
|
484 |
|
485 | Intended use is to place the following code in a package's __init__.py:
|
486 |
|
487 | from pkgutil import extend_path
|
488 | __path__ = extend_path(__path__, __name__)
|
489 |
|
490 | This will add to the package's __path__ all subdirectories of
|
491 | directories on sys.path named after the package. This is useful
|
492 | if one wants to distribute different parts of a single logical
|
493 | package as multiple directories.
|
494 |
|
495 | It also looks for *.pkg files beginning where * matches the name
|
496 | argument. This feature is similar to *.pth files (see site.py),
|
497 | except that it doesn't special-case lines starting with 'import'.
|
498 | A *.pkg file is trusted at face value: apart from checking for
|
499 | duplicates, all entries found in a *.pkg file are added to the
|
500 | path, regardless of whether they are exist the filesystem. (This
|
501 | is a feature.)
|
502 |
|
503 | If the input path is not a list (as is the case for frozen
|
504 | packages) it is returned unchanged. The input path is not
|
505 | modified; an extended copy is returned. Items are only appended
|
506 | to the copy at the end.
|
507 |
|
508 | It is assumed that sys.path is a sequence. Items of sys.path that
|
509 | are not (unicode or 8-bit) strings referring to existing
|
510 | directories are ignored. Unicode items of sys.path that cause
|
511 | errors when used as filenames may cause this function to raise an
|
512 | exception (in line with os.path.isdir() behavior).
|
513 | """
|
514 |
|
515 | if not isinstance(path, list):
|
516 | # This could happen e.g. when this is called from inside a
|
517 | # frozen package. Return the path unchanged in that case.
|
518 | return path
|
519 |
|
520 | pname = os.path.join(*name.split('.')) # Reconstitute as relative path
|
521 | # Just in case os.extsep != '.'
|
522 | sname = os.extsep.join(name.split('.'))
|
523 | sname_pkg = sname + os.extsep + "pkg"
|
524 | init_py = "__init__" + os.extsep + "py"
|
525 |
|
526 | path = path[:] # Start with a copy of the existing path
|
527 |
|
528 | for dir in sys.path:
|
529 | if not isinstance(dir, basestring) or not os.path.isdir(dir):
|
530 | continue
|
531 | subdir = os.path.join(dir, pname)
|
532 | # XXX This may still add duplicate entries to path on
|
533 | # case-insensitive filesystems
|
534 | initfile = os.path.join(subdir, init_py)
|
535 | if subdir not in path and os.path.isfile(initfile):
|
536 | path.append(subdir)
|
537 | # XXX Is this the right thing for subpackages like zope.app?
|
538 | # It looks for a file named "zope.app.pkg"
|
539 | pkgfile = os.path.join(dir, sname_pkg)
|
540 | if os.path.isfile(pkgfile):
|
541 | try:
|
542 | f = open(pkgfile)
|
543 | except IOError, msg:
|
544 | sys.stderr.write("Can't open %s: %s\n" %
|
545 | (pkgfile, msg))
|
546 | else:
|
547 | for line in f:
|
548 | line = line.rstrip('\n')
|
549 | if not line or line.startswith('#'):
|
550 | continue
|
551 | path.append(line) # Don't check for existence!
|
552 | f.close()
|
553 |
|
554 | return path
|
555 |
|
556 | def get_data(package, resource):
|
557 | """Get a resource from a package.
|
558 |
|
559 | This is a wrapper round the PEP 302 loader get_data API. The package
|
560 | argument should be the name of a package, in standard module format
|
561 | (foo.bar). The resource argument should be in the form of a relative
|
562 | filename, using '/' as the path separator. The parent directory name '..'
|
563 | is not allowed, and nor is a rooted name (starting with a '/').
|
564 |
|
565 | The function returns a binary string, which is the contents of the
|
566 | specified resource.
|
567 |
|
568 | For packages located in the filesystem, which have already been imported,
|
569 | this is the rough equivalent of
|
570 |
|
571 | d = os.path.dirname(sys.modules[package].__file__)
|
572 | data = open(os.path.join(d, resource), 'rb').read()
|
573 |
|
574 | If the package cannot be located or loaded, or it uses a PEP 302 loader
|
575 | which does not support get_data(), then None is returned.
|
576 | """
|
577 |
|
578 | loader = get_loader(package)
|
579 | if loader is None or not hasattr(loader, 'get_data'):
|
580 | return None
|
581 | mod = sys.modules.get(package) or loader.load_module(package)
|
582 | if mod is None or not hasattr(mod, '__file__'):
|
583 | return None
|
584 |
|
585 | # Modify the resource name to be compatible with the loader.get_data
|
586 | # signature - an os.path format "filename" starting with the dirname of
|
587 | # the package's __file__
|
588 | parts = resource.split('/')
|
589 | parts.insert(0, os.path.dirname(mod.__file__))
|
590 | resource_name = os.path.join(*parts)
|
591 | return loader.get_data(resource_name)
|