Application: gvSIG desktop » gvSIG scripting

# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

#

# This file is part of astroid.

#

# astroid is free software: you can redistribute it and/or modify it under

# the terms of the GNU Lesser General Public License as published by the Free

# Software Foundation, either version 2.1 of the License, or (at your option) any

# later version.

#

# astroid is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

# details.

#

# You should have received a copy of the GNU Lesser General Public License along

# with astroid.  If not, see <http://www.gnu.org/licenses/>.

"""Python modules manipulation utility functions.

:type PY_SOURCE_EXTS: tuple(str)

:var PY_SOURCE_EXTS: list of possible python source file extension

:type STD_LIB_DIRS: set of str

:var STD_LIB_DIRS: directories where standard modules are located

:type BUILTIN_MODULES: dict

:var BUILTIN_MODULES: dictionary with builtin module names has key

"""

from __future__ import with_statement

import imp

import os

import platform

import sys

from distutils.sysconfig import get_python_lib

from distutils.errors import DistutilsPlatformError

import zipimport

try:

    import pkg_resources

except ImportError:

    pkg_resources = None

PY_ZIPMODULE = object()

if sys.platform.startswith('win'):

    PY_SOURCE_EXTS = ('py', 'pyw')

    PY_COMPILED_EXTS = ('dll', 'pyd')

else:

    PY_SOURCE_EXTS = ('py',)

    PY_COMPILED_EXTS = ('so',)

try:

    # The explicit sys.prefix is to work around a patch in virtualenv that

    # replaces the 'real' sys.prefix (i.e. the location of the binary)

    # with the prefix from which the virtualenv was created. This throws

    # off the detection logic for standard library modules, thus the

    # workaround.

    STD_LIB_DIRS = set([

        get_python_lib(standard_lib=True, prefix=sys.prefix),

        # Take care of installations where exec_prefix != prefix.

        get_python_lib(standard_lib=True, prefix=sys.exec_prefix),

        get_python_lib(standard_lib=True)])

# get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to

# non-valid path, see https://bugs.pypy.org/issue1164

except DistutilsPlatformError:

    STD_LIB_DIRS = set()

if os.name == 'nt':

    STD_LIB_DIRS.add(os.path.join(sys.prefix, 'dlls'))

    try:

        # real_prefix is defined when running inside virtualenv.

        STD_LIB_DIRS.add(os.path.join(sys.real_prefix, 'dlls'))

    except AttributeError:

        pass

if platform.python_implementation() == 'PyPy':

    _root = os.path.join(sys.prefix, 'lib_pypy')

    STD_LIB_DIRS.add(_root)

    try:

        # real_prefix is defined when running inside virtualenv.

        STD_LIB_DIRS.add(os.path.join(sys.real_prefix, 'lib_pypy'))

    except AttributeError:

        pass

    del _root

if os.name == 'posix':

    # Need the real prefix is we're under a virtualenv, otherwise

    # the usual one will do.

    try:

        prefix = sys.real_prefix

    except AttributeError:

        prefix = sys.prefix

    def _posix_path(path):

        base_python = 'python%d.%d' % sys.version_info[:2]

        return os.path.join(prefix, path, base_python)

    STD_LIB_DIRS.add(_posix_path('lib'))

    if sys.maxsize > 2**32:

        # This tries to fix a problem with /usr/lib64 builds,

        # where systems are running both 32-bit and 64-bit code

        # on the same machine, which reflects into the places where

        # standard library could be found. More details can be found

        # here http://bugs.python.org/issue1294959.

        # An easy reproducing case would be

        # https://github.com/PyCQA/pylint/issues/712#issuecomment-163178753

        STD_LIB_DIRS.add(_posix_path('lib64'))

EXT_LIB_DIR = get_python_lib()

IS_JYTHON = platform.python_implementation() == 'Jython'

BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)

class NoSourceFile(Exception):

    """exception raised when we are not able to get a python

    source file for a precompiled file

    """

def _normalize_path(path):

    return os.path.normcase(os.path.abspath(path))

def _path_from_filename(filename, is_jython=IS_JYTHON):

    if not is_jython:

        if sys.version_info > (3, 0):

            return filename

        else:

            if filename.endswith(".pyc"):

                return filename[:-1]

            return filename

    head, has_pyclass, _ = filename.partition("$py.class")

    if has_pyclass:

        return head + ".py"

    return filename

def _handle_blacklist(blacklist, dirnames, filenames):

    """remove files/directories in the black list

    dirnames/filenames are usually from os.walk

    """

    for norecurs in blacklist:

        if norecurs in dirnames:

            dirnames.remove(norecurs)

        elif norecurs in filenames:

            filenames.remove(norecurs)

_NORM_PATH_CACHE = {}

def _cache_normalize_path(path):

    """abspath with caching"""

    # _module_file calls abspath on every path in sys.path every time it's

    # called; on a larger codebase this easily adds up to half a second just

    # assembling path components. This cache alleviates that.

    try:

        return _NORM_PATH_CACHE[path]

    except KeyError:

        if not path: # don't cache result for ''

            return _normalize_path(path)

        result = _NORM_PATH_CACHE[path] = _normalize_path(path)

        return result

def load_module_from_name(dotted_name, path=None, use_sys=True):

    """Load a Python module from its name.

    :type dotted_name: str

    :param dotted_name: python name of a module or package

    :type path: list or None

    :param path:

      optional list of path where the module or package should be

      searched (use sys.path if nothing or None is given)

    :type use_sys: bool

    :param use_sys:

      boolean indicating whether the sys.modules dictionary should be

      used or not

    :raise ImportError: if the module or package is not found

    :rtype: module

    :return: the loaded module

    """

    return load_module_from_modpath(dotted_name.split('.'), path, use_sys)

def load_module_from_modpath(parts, path=None, use_sys=1):

    """Load a python module from its splitted name.

    :type parts: list(str) or tuple(str)

    :param parts:

      python name of a module or package splitted on '.'

    :type path: list or None

    :param path:

      optional list of path where the module or package should be

      searched (use sys.path if nothing or None is given)

    :type use_sys: bool

    :param use_sys:

      boolean indicating whether the sys.modules dictionary should be used or not

    :raise ImportError: if the module or package is not found

    :rtype: module

    :return: the loaded module

    """

    if use_sys:

        try:

            return sys.modules['.'.join(parts)]

        except KeyError:

            pass

    modpath = []

    prevmodule = None

    for part in parts:

        modpath.append(part)

        curname = '.'.join(modpath)

        module = None

        if len(modpath) != len(parts):

            # even with use_sys=False, should try to get outer packages from sys.modules

            module = sys.modules.get(curname)

        elif use_sys:

            # because it may have been indirectly loaded through a parent

            module = sys.modules.get(curname)

        if module is None:

            mp_file, mp_filename, mp_desc = imp.find_module(part, path)

            module = imp.load_module(curname, mp_file, mp_filename, mp_desc)

            # mp_file still needs to be closed.

            if mp_file:

                mp_file.close()

        if prevmodule:

            setattr(prevmodule, part, module)

        _file = getattr(module, '__file__', '')

        prevmodule = module

        if not _file and _is_namespace(curname):

            continue

        if not _file and len(modpath) != len(parts):

            raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]))

        path = [os.path.dirname(_file)]

    return module

def load_module_from_file(filepath, path=None, use_sys=True, extrapath=None):

    """Load a Python module from it's path.

    :type filepath: str

    :param filepath: path to the python module or package

    :type path: list or None

    :param path:

      optional list of path where the module or package should be

      searched (use sys.path if nothing or None is given)

    :type use_sys: bool

    :param use_sys:

      boolean indicating whether the sys.modules dictionary should be

      used or not

    :raise ImportError: if the module or package is not found

    :rtype: module

    :return: the loaded module

    """

    modpath = modpath_from_file(filepath, extrapath)

    return load_module_from_modpath(modpath, path, use_sys)

def _check_init(path, mod_path):

    """check there are some __init__.py all along the way"""

    modpath = []

    for part in mod_path:

        modpath.append(part)

        path = os.path.join(path, part)

        if not _is_namespace('.'.join(modpath)) and not _has_init(path):

            return False

    return True

def modpath_from_file(filename, extrapath=None):

    """given a file path return the corresponding splitted module's name

    (i.e name of a module or package splitted on '.')

    :type filename: str

    :param filename: file's path for which we want the module's name

    :type extrapath: dict

    :param extrapath:

      optional extra search path, with path as key and package name for the path

      as value. This is usually useful to handle package splitted in multiple

      directories using __path__ trick.

    :raise ImportError:

      if the corresponding module's name has not been found

    :rtype: list(str)

    :return: the corresponding splitted module's name

    """

    filename = _path_from_filename(filename)

    filename = os.path.abspath(filename)

    base = os.path.splitext(filename)[0]

    if extrapath is not None:

        for path_ in extrapath:

            path = os.path.abspath(path_)

            if path and os.path.normcase(base[:len(path)]) == os.path.normcase(path):

                submodpath = [pkg for pkg in base[len(path):].split(os.sep)

                              if pkg]

                if _check_init(path, submodpath[:-1]):

                    return extrapath[path_].split('.') + submodpath

    for path in sys.path:

        path = _cache_normalize_path(path)

        if path and os.path.normcase(base).startswith(path):

            modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]

            if _check_init(path, modpath[:-1]):

                return modpath

    raise ImportError('Unable to find module for %s in %s' % (

        filename, ', \n'.join(sys.path)))

def file_from_modpath(modpath, path=None, context_file=None):

    return file_info_from_modpath(modpath, path, context_file)[0]

def file_info_from_modpath(modpath, path=None, context_file=None):

    """given a mod path (i.e. splitted module / package name), return the

    corresponding file, giving priority to source file over precompiled

    file if it exists

    :type modpath: list or tuple

    :param modpath:

      splitted module's name (i.e name of a module or package splitted

      on '.')

      (this means explicit relative imports that start with dots have

      empty strings in this list!)

    :type path: list or None

    :param path:

      optional list of path where the module or package should be

      searched (use sys.path if nothing or None is given)

    :type context_file: str or None

    :param context_file:

      context file to consider, necessary if the identifier has been

      introduced using a relative import unresolvable in the actual

      context (i.e. modutils)

    :raise ImportError: if there is no such module in the directory

    :rtype: (str or None, import type)

    :return:

      the path to the module's file or None if it's an integrated

      builtin module such as 'sys'

    """

    if context_file is not None:

        context = os.path.dirname(context_file)

    else:

        context = context_file

    if modpath[0] == 'xml':

        # handle _xmlplus

        try:

            return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)

        except ImportError:

            return _file_from_modpath(modpath, path, context)

    elif modpath == ['os', 'path']:

        # FIXME: currently ignoring search_path...

        return os.path.__file__, imp.PY_SOURCE

    return _file_from_modpath(modpath, path, context)

def get_module_part(dotted_name, context_file=None):

    """given a dotted name return the module part of the name :

    >>> get_module_part('astroid.as_string.dump')

    'astroid.as_string'

    :type dotted_name: str

    :param dotted_name: full name of the identifier we are interested in

    :type context_file: str or None

    :param context_file:

      context file to consider, necessary if the identifier has been

      introduced using a relative import unresolvable in the actual

      context (i.e. modutils)

    :raise ImportError: if there is no such module in the directory

    :rtype: str or None

    :return:

      the module part of the name or None if we have not been able at

      all to import the given name

    XXX: deprecated, since it doesn't handle package precedence over module

    (see #10066)

    """

    # os.path trick

    if dotted_name.startswith('os.path'):

        return 'os.path'

    parts = dotted_name.split('.')

    if context_file is not None:

        # first check for builtin module which won't be considered latter

        # in that case (path != None)

        if parts[0] in BUILTIN_MODULES:

            if len(parts) > 2:

                raise ImportError(dotted_name)

            return parts[0]

        # don't use += or insert, we want a new list to be created !

    path = None

    starti = 0

    if parts[0] == '':

        assert context_file is not None, \

                'explicit relative import, but no context_file?'

        path = [] # prevent resolving the import non-relatively

        starti = 1

    while parts[starti] == '': # for all further dots: change context

        starti += 1

        context_file = os.path.dirname(context_file)

    for i in range(starti, len(parts)):

        try:

            file_from_modpath(parts[starti:i+1], path=path,

                              context_file=context_file)

        except ImportError:

            if not i >= max(1, len(parts) - 2):

                raise

            return '.'.join(parts[:i])

    return dotted_name

def get_module_files(src_directory, blacklist):

    """given a package directory return a list of all available python

    module's files in the package and its subpackages

    :type src_directory: str

    :param src_directory:

      path of the directory corresponding to the package

    :type blacklist: list or tuple

    :param blacklist: iterable

      list of files or directories to ignore.

    :rtype: list

    :return:

      the list of all available python module's files in the package and

      its subpackages

    """

    files = []

    for directory, dirnames, filenames in os.walk(src_directory):

        _handle_blacklist(blacklist, dirnames, filenames)

        # check for __init__.py

        if not '__init__.py' in filenames:

            dirnames[:] = ()

            continue

        for filename in filenames:

            if _is_python_file(filename):

                src = os.path.join(directory, filename)

                files.append(src)

    return files

def get_source_file(filename, include_no_ext=False):

    """given a python module's file name return the matching source file

    name (the filename will be returned identically if it's a already an

    absolute path to a python source file...)

    :type filename: str

    :param filename: python module's file name

    :raise NoSourceFile: if no source file exists on the file system

    :rtype: str

    :return: the absolute path of the source file if it exists

    """

    copy = getattr(filename,"copy",None)

    filename = os.path.abspath(_path_from_filename(filename))

    base, orig_ext = os.path.splitext(filename)

    for ext in PY_SOURCE_EXTS:

        source_path = '%s.%s' % (base, ext)

        if os.path.exists(source_path):

            if copy==None:

              return source_path

            return copy(source_path)

    if include_no_ext and not orig_ext and os.path.exists(base):

        return base

    raise NoSourceFile(filename)

def is_python_source(filename):

    """

    rtype: bool

    return: True if the filename is a python source file

    """

    return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS

def is_standard_module(modname, std_path=None):

    """try to guess if a module is a standard python module (by default,

    see `std_path` parameter's description)

    :type modname: str

    :param modname: name of the module we are interested in

    :type std_path: list(str) or tuple(str)

    :param std_path: list of path considered has standard

    :rtype: bool

    :return:

      true if the module:

      - is located on the path listed in one of the directory in `std_path`

      - is a built-in module

    """

    modname = modname.split('.')[0]

    try:

        filename = file_from_modpath([modname])

    except ImportError:

        # import failed, i'm probably not so wrong by supposing it's

        # not standard...

        return False

    # modules which are not living in a file are considered standard

    # (sys and __builtin__ for instance)

    if filename is None:

        # we assume there are no namespaces in stdlib

        return not _is_namespace(modname)

    filename = _normalize_path(filename)

    if filename.startswith(_cache_normalize_path(EXT_LIB_DIR)):

        return False

    if std_path is None:

        std_path = STD_LIB_DIRS

    for path in std_path:

        if filename.startswith(_cache_normalize_path(path)):

            return True

    return False

def is_relative(modname, from_file):

    """return true if the given module name is relative to the given

    file name

    :type modname: str

    :param modname: name of the module we are interested in

    :type from_file: str

    :param from_file:

      path of the module from which modname has been imported

    :rtype: bool

    :return:

      true if the module has been imported relatively to `from_file`

    """

    if not os.path.isdir(from_file):

        from_file = os.path.dirname(from_file)

    if from_file in sys.path:

        return False

    try:

        stream, _, _ = imp.find_module(modname.split('.')[0], [from_file])

        # Close the stream to avoid ResourceWarnings.

        if stream:

            stream.close()

        return True

    except ImportError:

        return False

# internal only functions #####################################################

def _file_from_modpath(modpath, path=None, context=None):

    """given a mod path (i.e. splitted module / package name), return the

    corresponding file

    this function is used internally, see `file_from_modpath`'s

    documentation for more information

    """

    assert len(modpath) > 0

    if context is not None:

        try:

            mtype, mp_filename = _module_file(modpath, [context])

        except ImportError:

            mtype, mp_filename = _module_file(modpath, path)

    else:

        mtype, mp_filename = _module_file(modpath, path)

    if mtype == imp.PY_COMPILED:

        try:

            return get_source_file(mp_filename), imp.PY_SOURCE

        except NoSourceFile:

            return mp_filename, imp.PY_COMPILED

    elif mtype == imp.C_BUILTIN:

        # integrated builtin module

        return None, imp.C_BUILTIN

    elif mtype == imp.PKG_DIRECTORY:

        mp_filename = _has_init(mp_filename)

        mtype = imp.PY_SOURCE

    return mp_filename, mtype

def _search_zip(modpath, pic):

    for filepath, importer in list(pic.items()):

        if importer is not None:

            if importer.find_module(modpath[0]):

                if not importer.find_module(os.path.sep.join(modpath)):

                    raise ImportError('No module named %s in %s/%s' % (

                        '.'.join(modpath[1:]), filepath, modpath))

                return (PY_ZIPMODULE,

                        os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath),

                        filepath)

    raise ImportError('No module named %s' % '.'.join(modpath))

try:

    import pkg_resources

except ImportError:

    pkg_resources = None

def _is_namespace(modname):

    return (pkg_resources is not None

            and modname in pkg_resources._namespace_packages)

def _module_file(modpath, path=None):

    """get a module type / file path

    :type modpath: list or tuple

    :param modpath:

      splitted module's name (i.e name of a module or package splitted

      on '.'), with leading empty strings for explicit relative import

    :type path: list or None

    :param path:

      optional list of path where the module or package should be

      searched (use sys.path if nothing or None is given)

    :rtype: tuple(int, str)

    :return: the module type flag and the file path for a module

    """

    # egg support compat

    try:

        pic = sys.path_importer_cache

        _path = (path is None and sys.path or path)

        for __path in _path:

            if not __path in pic:

                try:

                    pic[__path] = zipimport.zipimporter(__path)

                except zipimport.ZipImportError:

                    pic[__path] = None

        checkeggs = True

    except AttributeError:

        checkeggs = False

    # pkg_resources support (aka setuptools namespace packages)

    if _is_namespace(modpath[0]) and modpath[0] in sys.modules:

        # setuptools has added into sys.modules a module object with proper

        # __path__, get back information from there

        module = sys.modules[modpath.pop(0)]

        path = module.__path__

        if not modpath:

            return imp.C_BUILTIN, None

    imported = []

    while modpath:

        modname = modpath[0]

        # take care to changes in find_module implementation wrt builtin modules

        #

        # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23)

        # >>> imp.find_module('posix')

        # (None, 'posix', ('', '', 6))

        #

        # Python 3.3.1 (default, Apr 26 2013, 12:08:46)

        # >>> imp.find_module('posix')

        # (None, None, ('', '', 6))

        try:

            stream, mp_filename, mp_desc = imp.find_module(modname, path)

        except ImportError:

            if checkeggs:

                return _search_zip(modpath, pic)[:2]

            raise

        else:

            # Don't forget to close the stream to avoid

            # spurious ResourceWarnings.

            if stream:

                stream.close()

            if checkeggs and mp_filename:

                fullabspath = [_cache_normalize_path(x) for x in _path]

                try:

                    pathindex = fullabspath.index(os.path.dirname(_normalize_path(mp_filename)))

                    emtype, emp_filename, zippath = _search_zip(modpath, pic)

                    if pathindex > _path.index(zippath):

                        # an egg takes priority

                        return emtype, emp_filename

                except ValueError:

                    # XXX not in _path

                    pass

                except ImportError:

                    pass

                checkeggs = False

        imported.append(modpath.pop(0))

        mtype = mp_desc[2]

        if modpath:

            if mtype != imp.PKG_DIRECTORY:

                raise ImportError('No module %s in %s' % ('.'.join(modpath),

                                                          '.'.join(imported)))

            # XXX guess if package is using pkgutil.extend_path by looking for

            # those keywords in the first four Kbytes

            try:

                with open(os.path.join(mp_filename, '__init__.py'), 'rb') as stream:

                    data = stream.read(4096)

            except IOError:

                path = [mp_filename]

            else:

                extend_path = b'pkgutil' in data and b'extend_path' in data

                declare_namespace = (

                    b"pkg_resources" in data

                    and b"declare_namespace(__name__)" in data)

                if extend_path or declare_namespace:

                    # extend_path is called, search sys.path for module/packages

                    # of this name see pkgutil.extend_path documentation

                    path = [os.path.join(p, *imported) for p in sys.path

                            if os.path.isdir(os.path.join(p, *imported))]

                else:

                    path = [mp_filename]

    return mtype, mp_filename

def _is_python_file(filename):

    """return true if the given filename should be considered as a python file

    .pyc and .pyo are ignored

    """

    for ext in ('.py', '.so', '.pyd', '.pyw'):

        if filename.endswith(ext):

            return True

    return False

def _has_init(directory):

    """if the given directory has a valid __init__ file, return its path,

    else return None

    """

    mod_or_pack = os.path.join(directory, '__init__')

    for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):

        if os.path.exists(mod_or_pack + '.' + ext):

            return mod_or_pack + '.' + ext

    return None