Application: gvSIG desktop » gvSIG scripting

# Copyright (c) 2003-2013 LOGILAB S.A. (Paris, FRANCE).

# http://www.logilab.fr/ -- mailto:contact@logilab.fr

#

# This program is free software; you can redistribute it and/or modify it under

# the terms of the GNU General Public License as published by the Free Software

# Foundation; either version 2 of the License, or (at your option) any later

# version.

#

# This program is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License along with

# this program; if not, write to the Free Software Foundation, Inc.,

# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

"""imports checkers for Python code"""

import collections

from distutils import sysconfig

import os

import sys

import six

import astroid

from astroid import are_exclusive

from astroid.modutils import (get_module_part, is_standard_module,

                              file_from_modpath)

from pylint.interfaces import IAstroidChecker

from pylint.utils import EmptyReport, get_global_option

from pylint.checkers import BaseChecker

from pylint.checkers.utils import check_messages, node_ignores_exception

from pylint.graph import get_cycles, DotBackend

from pylint.reporters.ureports.nodes import VerbatimText, Paragraph

def _qualified_names(modname):

    """Split the names of the given module into subparts

    For example,

        _qualified_names('pylint.checkers.ImportsChecker')

    returns

        ['pylint', 'pylint.checkers', 'pylint.checkers.ImportsChecker']

    """

    names = modname.split('.')

    return ['.'.join(names[0:i+1]) for i in range(len(names))]

def _get_import_name(importnode, modname):

    """Get a prepared module name from the given import node

    In the case of relative imports, this will return the

    absolute qualified module name, which might be useful

    for debugging. Otherwise, the initial module name

    is returned unchanged.

    """

    if isinstance(importnode, astroid.ImportFrom):

        if importnode.level:

            root = importnode.root()

            if isinstance(root, astroid.Module):

                modname = root.relative_to_absolute_name(

                    modname, level=importnode.level)

    return modname

def _get_first_import(node, context, name, base, level):

    """return the node where [base.]<name> is imported or None if not found

    """

    fullname = '%s.%s' % (base, name) if base else name

    first = None

    found = False

    for first in context.body:

        if first is node:

            continue

        if first.scope() is node.scope() and first.fromlineno > node.fromlineno:

            continue

        if isinstance(first, astroid.Import):

            if any(fullname == iname[0] for iname in first.names):

                found = True

                break

        elif isinstance(first, astroid.ImportFrom):

            if level == first.level and any(

                    fullname == '%s.%s' % (first.modname, iname[0])

                    for iname in first.names):

                found = True

                break

    if found and not are_exclusive(first, node):

        return first

# utilities to represents import dependencies as tree and dot graph ###########

def _make_tree_defs(mod_files_list):

    """get a list of 2-uple (module, list_of_files_which_import_this_module),

    it will return a dictionary to represent this as a tree

    """

    tree_defs = {}

    for mod, files in mod_files_list:

        node = (tree_defs, ())

        for prefix in mod.split('.'):

            node = node[0].setdefault(prefix, [{}, []])

        node[1] += files

    return tree_defs

def _repr_tree_defs(data, indent_str=None):

    """return a string which represents imports as a tree"""

    lines = []

    nodes = data.items()

    for i, (mod, (sub, files)) in enumerate(sorted(nodes, key=lambda x: x[0])):

        if not files:

            files = ''

        else:

            files = '(%s)' % ','.join(files)

        if indent_str is None:

            lines.append('%s %s' % (mod, files))

            sub_indent_str = '  '

        else:

            lines.append(r'%s\-%s %s' % (indent_str, mod, files))

            if i == len(nodes)-1:

                sub_indent_str = '%s  ' % indent_str

            else:

                sub_indent_str = '%s| ' % indent_str

        if sub:

            lines.append(_repr_tree_defs(sub, sub_indent_str))

    return '\n'.join(lines)

def _dependencies_graph(filename, dep_info):

    """write dependencies as a dot (graphviz) file

    """

    done = {}

    printer = DotBackend(filename[:-4], rankdir='LR')

    printer.emit('URL="." node[shape="box"]')

    for modname, dependencies in sorted(six.iteritems(dep_info)):

        done[modname] = 1

        printer.emit_node(modname)

        for modname in dependencies:

            if modname not in done:

                done[modname] = 1

                printer.emit_node(modname)

    for depmodname, dependencies in sorted(six.iteritems(dep_info)):

        for modname in dependencies:

            printer.emit_edge(modname, depmodname)

    printer.generate(filename)

def _make_graph(filename, dep_info, sect, gtype):

    """generate a dependencies graph and add some information about it in the

    report's section

    """

    _dependencies_graph(filename, dep_info)

    sect.append(Paragraph('%simports graph has been written to %s'

                          % (gtype, filename)))

# the import checker itself ###################################################

MSGS = {

    'E0401': ('Unable to import %s',

              'import-error',

              'Used when pylint has been unable to import a module.',

              {'old_names': [('F0401', 'import-error')]}),

    'R0401': ('Cyclic import (%s)',

              'cyclic-import',

              'Used when a cyclic import between two or more modules is \

              detected.'),

    'W0401': ('Wildcard import %s',

              'wildcard-import',

              'Used when `from module import *` is detected.'),

    'W0402': ('Uses of a deprecated module %r',

              'deprecated-module',

              'Used a module marked as deprecated is imported.'),

    'W0403': ('Relative import %r, should be %r',

              'relative-import',

              'Used when an import relative to the package directory is '

              'detected.',

              {'maxversion': (3, 0)}),

    'W0404': ('Reimport %r (imported line %s)',

              'reimported',

              'Used when a module is reimported multiple times.'),

    'W0406': ('Module import itself',

              'import-self',

              'Used when a module is importing itself.'),

    'W0410': ('__future__ import is not the first non docstring statement',

              'misplaced-future',

              'Python 2.5 and greater require __future__ import to be the \

              first non docstring statement in the module.'),

    'C0410': ('Multiple imports on one line (%s)',

              'multiple-imports',

              'Used when import statement importing multiple modules is '

              'detected.'),

    'C0411': ('%s comes before %s',

              'wrong-import-order',

              'Used when PEP8 import order is not respected (standard imports '

              'first, then third-party libraries, then local imports)'),

    'C0412': ('Imports from package %s are not grouped',

              'ungrouped-imports',

              'Used when imports are not grouped by packages'),

    'C0413': ('Import "%s" should be placed at the top of the '

              'module',

              'wrong-import-position',

              'Used when code and imports are mixed'),

    }

class ImportsChecker(BaseChecker):

    """checks for

    * external modules dependencies

    * relative / wildcard imports

    * cyclic imports

    * uses of deprecated modules

    """

    __implements__ = IAstroidChecker

    name = 'imports'

    msgs = MSGS

    priority = -2

    if six.PY2:

        deprecated_modules = ('regsub', 'TERMIOS', 'Bastion', 'rexec')

    else:

        deprecated_modules = ('optparse', )

    options = (('deprecated-modules',

                {'default' : deprecated_modules,

                 'type' : 'csv',

                 'metavar' : '<modules>',

                 'help' : 'Deprecated modules which should not be used, \

separated by a comma'}

               ),

               ('import-graph',

                {'default' : '',

                 'type' : 'string',

                 'metavar' : '<file.dot>',

                 'help' : 'Create a graph of every (i.e. internal and \

external) dependencies in the given file (report RP0402 must not be disabled)'}

               ),

               ('ext-import-graph',

                {'default' : '',

                 'type' : 'string',

                 'metavar' : '<file.dot>',

                 'help' : 'Create a graph of external dependencies in the \

given file (report RP0402 must not be disabled)'}

               ),

               ('int-import-graph',

                {'default' : '',

                 'type' : 'string',

                 'metavar' : '<file.dot>',

                 'help' : 'Create a graph of internal dependencies in the \

given file (report RP0402 must not be disabled)'}

               ),

              )

    def __init__(self, linter=None):

        BaseChecker.__init__(self, linter)

        self.stats = None

        self.import_graph = None

        self._imports_stack = []

        self._first_non_import_node = None

        self.__int_dep_info = self.__ext_dep_info = None

        self.reports = (('RP0401', 'External dependencies',

                         self._report_external_dependencies),

                        ('RP0402', 'Modules dependencies graph',

                         self._report_dependencies_graph),

                       )

        self._site_packages = self._compute_site_packages()

    @staticmethod

    def _compute_site_packages():

        def _normalized_path(path):

            return os.path.normcase(os.path.abspath(path))

        paths = set()

        real_prefix = getattr(sys, 'real_prefix', None)

        for prefix in filter(None, (real_prefix, sys.prefix)):

            path = sysconfig.get_python_lib(prefix=prefix)

            path = _normalized_path(path)

            paths.add(path)

        # Handle Debian's derivatives /usr/local.

        if os.path.isfile("/etc/debian_version"):

            for prefix in filter(None, (real_prefix, sys.prefix)):

                libpython = os.path.join(prefix, "local", "lib",

                                         "python" + sysconfig.get_python_version(),

                                         "dist-packages")

                paths.add(libpython)

        return paths

    def open(self):

        """called before visiting project (i.e set of modules)"""

        self.linter.add_stats(dependencies={})

        self.linter.add_stats(cycles=[])

        self.stats = self.linter.stats

        self.import_graph = collections.defaultdict(set)

        self._ignored_modules = get_global_option(

            self, 'ignored-modules', default=[])

    def close(self):

        """called before visiting project (i.e set of modules)"""

        # don't try to compute cycles if the associated message is disabled

        if self.linter.is_message_enabled('cyclic-import'):

            vertices = list(self.import_graph)

            for cycle in get_cycles(self.import_graph, vertices=vertices):

                self.add_message('cyclic-import', args=' -> '.join(cycle))

    @check_messages('wrong-import-position', 'multiple-imports',

                    'relative-import', 'reimported')

    def visit_import(self, node):

        """triggered when an import statement is seen"""

        self._check_reimport(node)

        modnode = node.root()

        names = [name for name, _ in node.names]

        if len(names) >= 2:

            self.add_message('multiple-imports', args=', '.join(names), node=node)

        for name in names:

            self._check_deprecated_module(node, name)

            importedmodnode = self.get_imported_module(node, name)

            if isinstance(node.scope(), astroid.Module):

                self._check_position(node)

                self._record_import(node, importedmodnode)

            if importedmodnode is None:

                continue

            self._check_relative_import(modnode, node, importedmodnode, name)

            self._add_imported_module(node, importedmodnode.name)

    @check_messages(*(MSGS.keys()))

    def visit_importfrom(self, node):

        """triggered when a from statement is seen"""

        basename = node.modname

        self._check_misplaced_future(node)

        self._check_deprecated_module(node, basename)

        self._check_wildcard_imports(node)

        self._check_same_line_imports(node)

        self._check_reimport(node, basename=basename, level=node.level)

        modnode = node.root()

        importedmodnode = self.get_imported_module(node, basename)

        if isinstance(node.scope(), astroid.Module):

            self._check_position(node)

            self._record_import(node, importedmodnode)

        if importedmodnode is None:

            return

        self._check_relative_import(modnode, node, importedmodnode, basename)

        for name, _ in node.names:

            if name != '*':

                self._add_imported_module(node, '%s.%s' % (importedmodnode.name, name))

    @check_messages('wrong-import-order', 'ungrouped-imports',

                    'wrong-import-position')

    def leave_module(self, node):

        # Check imports are grouped by category (standard, 3rd party, local)

        std_imports, ext_imports, loc_imports = self._check_imports_order(node)

        # Check imports are grouped by package within a given category

        met = set()

        current_package = None

        for import_node, import_name in std_imports + ext_imports + loc_imports:

            package, _, _ = import_name.partition('.')

            if current_package and current_package != package and package in met:

                self.add_message('ungrouped-imports', node=import_node,

                                 args=package)

            current_package = package

            met.add(package)

        self._imports_stack = []

        self._first_non_import_node = None

    def visit_if(self, node):

        # if the node does not contain an import instruction, and if it is the

        # first node of the module, keep a track of it (all the import positions

        # of the module will be compared to the position of this first

        # instruction)

        if self._first_non_import_node:

            return

        if not isinstance(node.parent, astroid.Module):

            return

        if any(node.nodes_of_class((astroid.Import, astroid.ImportFrom))):

            return

        self._first_non_import_node = node

    visit_tryfinally = visit_tryexcept = visit_assignattr = visit_assign \

            = visit_ifexp = visit_comprehension = visit_if

    def visit_functiondef(self, node):

        # If it is the first non import instruction of the module, record it.

        if self._first_non_import_node:

            return

        # Check if the node belongs to an `If` or a `Try` block. If they

        # contain imports, skip recording this node.

        if not isinstance(node.parent.scope(), astroid.Module):

            return

        root = node

        while not isinstance(root.parent, astroid.Module):

            root = root.parent

        if isinstance(root, (astroid.If, astroid.TryFinally, astroid.TryExcept)):

            if any(root.nodes_of_class((astroid.Import, astroid.ImportFrom))):

                return

        self._first_non_import_node = node

    visit_classdef = visit_for = visit_while = visit_functiondef

    def _check_misplaced_future(self, node):

        basename = node.modname

        if basename == '__future__':

            # check if this is the first non-docstring statement in the module

            prev = node.previous_sibling()

            if prev:

                # consecutive future statements are possible

                if not (isinstance(prev, astroid.ImportFrom)

                        and prev.modname == '__future__'):

                    self.add_message('misplaced-future', node=node)

            return

    def _check_same_line_imports(self, node):

        # Detect duplicate imports on the same line.

        names = (name for name, _ in node.names)

        counter = collections.Counter(names)

        for name, count in counter.items():

            if count > 1:

                self.add_message('reimported', node=node,

                                 args=(name, node.fromlineno))

    def _check_position(self, node):

        """Check `node` import or importfrom node position is correct

        Send a message  if `node` comes before another instruction

        """

        # if a first non-import instruction has already been encountered,

        # it means the import comes after it and therefore is not well placed

        if self._first_non_import_node:

            self.add_message('wrong-import-position', node=node,

                             args=node.as_string())

    def _record_import(self, node, importedmodnode):

        """Record the package `node` imports from"""

        importedname = importedmodnode.name if importedmodnode else None

        if not importedname:

            importedname = node.names[0][0].split('.')[0]

        self._imports_stack.append((node, importedname))

    @staticmethod

    def _is_fallback_import(node, imports):

        imports = [import_node for (import_node, _) in imports]

        return any(astroid.are_exclusive(import_node, node)

                   for import_node in imports)

    def _check_imports_order(self, node):

        """Checks imports of module `node` are grouped by category

        Imports must follow this order: standard, 3rd party, local

        """

        extern_imports = []

        local_imports = []

        std_imports = []

        for node, modname in self._imports_stack:

            package = modname.split('.')[0]

            if is_standard_module(modname):

                std_imports.append((node, package))

                wrong_import = extern_imports or local_imports

                if not wrong_import:

                    continue

                if self._is_fallback_import(node, wrong_import):

                    continue

                self.add_message('wrong-import-order', node=node,

                                 args=('standard import "%s"' % node.as_string(),

                                       '"%s"' % wrong_import[0][0].as_string()))

            else:

                try:

                    filename = file_from_modpath([package])

                except ImportError:

                    continue

                if not filename:

                    continue

                filename = os.path.normcase(os.path.abspath(filename))

                if not any(filename.startswith(path) for path in self._site_packages):

                    local_imports.append((node, package))

                    continue

                extern_imports.append((node, package))

                if not local_imports:

                    continue

                self.add_message('wrong-import-order', node=node,

                                 args=('external import "%s"' % node.as_string(),

                                       '"%s"' % local_imports[0][0].as_string()))

        return std_imports, extern_imports, local_imports

    def get_imported_module(self, importnode, modname):

        try:

            return importnode.do_import_module(modname)

        except astroid.InferenceError as ex:

            dotted_modname = _get_import_name(importnode, modname)

            if str(ex) != modname:

                args = '%r (%s)' % (dotted_modname, ex)

            else:

                args = repr(dotted_modname)

            for submodule in _qualified_names(modname):

                if submodule in self._ignored_modules:

                    return None

            if not node_ignores_exception(importnode, ImportError):

                self.add_message("import-error", args=args, node=importnode)

    def _check_relative_import(self, modnode, importnode, importedmodnode,

                               importedasname):

        """check relative import. node is either an Import or From node, modname

        the imported module name.

        """

        if not self.linter.is_message_enabled('relative-import'):

            return

        if importedmodnode.file is None:

            return False # built-in module

        if modnode is importedmodnode:

            return False # module importing itself

        if modnode.absolute_import_activated() or getattr(importnode, 'level', None):

            return False

        if importedmodnode.name != importedasname:

            # this must be a relative import...

            self.add_message('relative-import',

                             args=(importedasname, importedmodnode.name),

                             node=importnode)

    def _add_imported_module(self, node, importedmodname):

        """notify an imported module, used to analyze dependencies"""

        module_file = node.root().file

        context_name = node.root().name

        base = os.path.splitext(os.path.basename(module_file))[0]

        # Determine if we have a `from .something import` in a package's

        # __init__. This means the module will never be able to import

        # itself using this condition (the level will be bigger or

        # if the same module is named as the package, it will be different

        # anyway).

        if isinstance(node, astroid.ImportFrom):

            if node.level and node.level > 0 and base == '__init__':

                return

        try:

            importedmodname = get_module_part(importedmodname,

                                              module_file)

        except ImportError:

            pass

        if context_name == importedmodname:

            self.add_message('import-self', node=node)

        elif not is_standard_module(importedmodname):

            # handle dependencies

            importedmodnames = self.stats['dependencies'].setdefault(

                importedmodname, set())

            if context_name not in importedmodnames:

                importedmodnames.add(context_name)

            # update import graph

            mgraph = self.import_graph[context_name]

            if importedmodname not in mgraph:

                mgraph.add(importedmodname)

    def _check_deprecated_module(self, node, mod_path):

        """check if the module is deprecated"""

        for mod_name in self.config.deprecated_modules:

            if mod_path == mod_name or mod_path.startswith(mod_name + '.'):

                self.add_message('deprecated-module', node=node, args=mod_path)

    def _check_reimport(self, node, basename=None, level=None):

        """check if the import is necessary (i.e. not already done)"""

        if not self.linter.is_message_enabled('reimported'):

            return

        frame = node.frame()

        root = node.root()

        contexts = [(frame, level)]

        if root is not frame:

            contexts.append((root, None))

        for context, level in contexts:

            for name, _ in node.names:

                first = _get_first_import(node, context, name, basename, level)

                if first is not None:

                    self.add_message('reimported', node=node,

                                     args=(name, first.fromlineno))

    def _report_external_dependencies(self, sect, _, dummy):

        """return a verbatim layout for displaying dependencies"""

        dep_info = _make_tree_defs(six.iteritems(self._external_dependencies_info()))

        if not dep_info:

            raise EmptyReport()

        tree_str = _repr_tree_defs(dep_info)

        sect.append(VerbatimText(tree_str))

    def _report_dependencies_graph(self, sect, _, dummy):

        """write dependencies as a dot (graphviz) file"""

        dep_info = self.stats['dependencies']

        if not dep_info or not (self.config.import_graph

                                or self.config.ext_import_graph

                                or self.config.int_import_graph):

            raise EmptyReport()

        filename = self.config.import_graph

        if filename:

            _make_graph(filename, dep_info, sect, '')

        filename = self.config.ext_import_graph

        if filename:

            _make_graph(filename, self._external_dependencies_info(),

                        sect, 'external ')

        filename = self.config.int_import_graph

        if filename:

            _make_graph(filename, self._internal_dependencies_info(),

                        sect, 'internal ')

    def _external_dependencies_info(self):

        """return cached external dependencies information or build and

        cache them

        """

        if self.__ext_dep_info is None:

            package = self.linter.current_name

            self.__ext_dep_info = result = {}

            for importee, importers in six.iteritems(self.stats['dependencies']):

                if not importee.startswith(package):

                    result[importee] = importers

        return self.__ext_dep_info

    def _internal_dependencies_info(self):

        """return cached internal dependencies information or build and

        cache them

        """

        if self.__int_dep_info is None:

            package = self.linter.current_name

            self.__int_dep_info = result = {}

            for importee, importers in six.iteritems(self.stats['dependencies']):

                if importee.startswith(package):

                    result[importee] = importers

        return self.__int_dep_info

    def _check_wildcard_imports(self, node):

        for name, _ in node.names:

            if name == '*':

                self.add_message('wildcard-import', args=node.modname, node=node)

def register(linter):

    """required method to auto register this checker """

    linter.register_checker(ImportsChecker(linter))