pylint crashed with a AstroidError

[John2013]

Bug description

When linting, pylint crashed with a AstroidError and with the following stacktrace:

import csv
import logging

from django.core.management import BaseCommand
from django.db import transaction
from pik.core.shortcuts import update_or_create_object

from building_parts.models import BuildingPart, BuildingPartType
from core.utils.common import slugify_word
from housing.models import Building
from utility_systems.models import (
    UtilitySystemType, UtilitySystemExtraAttr, Elevator)

LOGGER = logging.getLogger(__name__)


def _read_rows_from_file(file_name):
    with open(file_name, encoding='utf-8') as csv_file:
        reader = csv.reader(csv_file, delimiter=';')
        for row in reader:
            yield row


class Command(BaseCommand):
    help = 'Upload elevators data'

    LIVING_ENTRANCE_TYPE_UID = '6e1b0154-4ac4-4511-81f7-ae80bdacc64f'
    ELEVATOR_TYPE_UID = '0167fa4a-f51f-1e4a-cfd3-7d6f33726faa'
    ACTIVITY_CODE_COLUMN = 3
    BUILDING_PART_NAME_COLUMN = 2
    ELEVATOR_NAME_COLUMN = 7
    ERP_CODE_COLUMN = 6

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.living_entrance_type = (
            BuildingPartType.objects.get(uid=self.LIVING_ENTRANCE_TYPE_UID))
        self.elevator_type = UtilitySystemType.objects.get(
            uid=self.ELEVATOR_TYPE_UID)
        self.headers = ()
        self.header_to_slug = {}
        self.extra_rows_start = 8

    def _is_valid_slug(self, slug):
        return UtilitySystemExtraAttr.objects.filter(
            slug=slug, utility_system_type=self.elevator_type).exists()

    def _validate_slugs(self, headers):
        header_to_slug = {}
        for header in headers:
            slug = slugify_word(header)
            if self._is_valid_slug(slug):
                header_to_slug[header] = slug
        return header_to_slug

    def _get_extra_attrs(self, extra_columns):
        result = {}
        for index, column in enumerate(
                extra_columns, start=self.extra_rows_start):
            header = self.headers[index]
            if header in self.header_to_slug:
                result[self.header_to_slug[header]] = column

        return result

    def _parse_row(self, row):
        activity_code = row[self.ACTIVITY_CODE_COLUMN].strip()
        building_part_name = row[self.BUILDING_PART_NAME_COLUMN]

        if building_part_name:
            building_part_name = f'№ {building_part_name.zfill(2)}'
        else:
            building_part_name = None

        elevator_name = row[self.ELEVATOR_NAME_COLUMN]
        extra_attrs = self._get_extra_attrs(row[self.extra_rows_start:])
        return {
            'activity_code': activity_code,
            'erp_code': row[self.ERP_CODE_COLUMN],
            'building_part_name': building_part_name,
            'elevator_name': elevator_name,
            'extra_attrs': extra_attrs}

    def _update(self, data):
        building_part_name = data['building_part_name']
        building_id = Building.objects.values_list('id', flat=True).get(
            activity_code=data['activity_code'])

        if building_part_name:
            building_part, _ = BuildingPart.objects.get_or_create(
                building_part_type=self.living_entrance_type,
                building_id=building_id, name=data['building_part_name'])
        else:
            building_part = None

        kwargs = {
            'utility_system_type': self.elevator_type,
            'building_part': building_part, 'building_id': building_id,
            'name': data['elevator_name']}
        elevator, _, _ = update_or_create_object(
            Elevator, search_keys={'erp_code': data['erp_code']}, **kwargs)

        elevator.extra_attrs.update(data['extra_attrs'])
        elevator.save()

    def add_arguments(self, parser):
        parser.add_argument('file_name', type=str)

    def handle(self, *args, **options):
        rows = list(_read_rows_from_file(options['file_name']))
        total_rows = len(rows) - 1
        LOGGER.debug(f'{total_rows}')
        self.headers = rows[0]
        self.header_to_slug = self._validate_slugs(self.headers)
        with transaction.atomic():
            Elevator.objects.all().delete()
            for line, row in enumerate(rows[1:]):
                LOGGER.debug(f'Line: {line}')
                data = self._parse_row(row)
                try:
                    self._update(data)
                except Building.DoesNotExist:
                    LOGGER.warning(
                        f"Building {data['activity_code']} doesn't exists!")
                    continue
                except Building.MultipleObjectsReturned:
                    LOGGER.warning(
                        f"Multiple buildings for {data['activity_code']} "
                        "returned!")
                    continue
                except Exception as exc:  # noqa: pylint=broad-except
                    LOGGER.error(exc)

pylint crashed with a AstroidError and with the following stacktrace:

Traceback (most recent call last):
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/lint/pylinter.py", line 790, in _lint_file
    check_astroid_module(module)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/lint/pylinter.py", line 1060, in check_astroid_module
    retval = self._check_astroid_module(
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/lint/pylinter.py", line 1110, in _check_astroid_module
    walker.walk(node)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/utils/ast_walker.py", line 93, in walk
    self.walk(child)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/utils/ast_walker.py", line 93, in walk
    self.walk(child)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/utils/ast_walker.py", line 93, in walk
    self.walk(child)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/utils/ast_walker.py", line 90, in walk
    callback(astroid)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 682, in visit_for
    self._check_unnecessary_list_index_lookup(node)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 2113, in _check_unnecessary_list_index_lookup
    has_start_arg, confidence = self._enumerate_with_start(node)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 2238, in _enumerate_with_start
    start_val, confidence = self._get_start_value(keyword.value)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 2255, in _get_start_value
    start_val = node.value
AttributeError: 'Attribute' object has no attribute 'value'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/lint/pylinter.py", line 755, in _lint_files
    self._lint_file(fileitem, module, check_astroid_module)
  File "/Users/timofeeves/projects/housing-service-backend/venv3/lib/python3.8/site-packages/pylint/lint/pylinter.py", line 792, in _lint_file
    raise astroid.AstroidError from e
astroid.exceptions.AstroidError

with pylint==2.15.5 its ok

Configuration

test-warnings: true
strictness: veryhigh

uses:
  - django
  - celery

pylint:
  options:
    django-settings-module: _project_.settings
  disable:
    - unused-argument
    - too-few-public-methods
    - too-many-arguments
    - too-many-ancestors
    - redefined-outer-name
    - relative-beyond-top-level
    - logging-fstring-interpolation
    - no-else-return
    - too-many-instance-attributes
    - not-an-iterable
    - too-many-lines
    - import-outside-toplevel
    - unnecessary-pass
    - invalid-str-returned
    - unnecessary-comprehension

Command used

prospector --profile-path . --profile .prospector.yaml foo.py

Pylint output

foo.py:1:1:
    L1:0 None: pylint - astroid-error
    foo.py: Fatal error while checking 'foo.py'. Please open an issue in our bug tracker so we address this. There is a pre-filled template that you can use in '/Users/timofeeves/Library/Caches/pylint/pylint-crash-2022-11-22-12-54-52.txt'.

Expected behavior

Pylint does not crash and shows its linting messages (if any) for foo.py

Pylint version

pylint 2.15.6
astroid 2.12.13
Python 3.8.10 (v3.8.10:3d8993a744, May  3 2021, 09:09:08) 
[Clang 12.0.5 (clang-1205.0.22.9)]

[Pierre-Sassoulas]

I can reproduce with:

class Command:
    def _get_extra_attrs(self, extra_columns):
        self.extra_rows_start = 8
        for index, column in enumerate(extra_columns, start=self.extra_rows_start):
            pass

It seems it's due to the false positive fix on enumerate with start, do you want to take a look @clavedeluna ?

[clavedeluna]

always 😢 to see your recently added code buggy. I'll take a look as soon as I can.

[XuehaiPan]

I can reproduce this with enumerate(..., start=<non-constexpr>):

y_start = 2
l = list(range(20))
for y, x in enumerate(l, start=y_start + 1):
    pass

************* Module test
test.py:1:0: C0114: Missing module docstring (missing-module-docstring)
test.py:1:0: C0103: Constant name "y_start" doesn't conform to UPPER_CASE naming style (invalid-name)
Exception on node <For l.3 at 0x10de01950> in file '/Users/PanXuehai/test.py'
Traceback (most recent call last):
  File "/usr/local/Cellar/pylint/2.15.6/libexec/lib/python3.11/site-packages/pylint/utils/ast_walker.py", line 90, in walk
    callback(astroid)
  File "/usr/local/Cellar/pylint/2.15.6/libexec/lib/python3.11/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 682, in visit_for
    self._check_unnecessary_list_index_lookup(node)
  File "/usr/local/Cellar/pylint/2.15.6/libexec/lib/python3.11/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 2113, in _check_unnecessary_list_index_lookup
    has_start_arg, confidence = self._enumerate_with_start(node)
                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Cellar/pylint/2.15.6/libexec/lib/python3.11/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 2238, in _enumerate_with_start
    start_val, confidence = self._get_start_value(keyword.value)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Cellar/pylint/2.15.6/libexec/lib/python3.11/site-packages/pylint/checkers/refactoring/refactoring_checker.py", line 2255, in _get_start_value
    start_val = node.value
                ^^^^^^^^^^
AttributeError: 'BinOp' object has no attribute 'value'
test.py:1:0: F0002: test.py: Fatal error while checking 'test.py'. Please open an issue in our bug tracker so we address this. There is a pre-filled template that you can use in '/Users/PanXuehai/Library/Caches/pylint/pylint-crash-2022-11-24-23-24-34.txt'. (astroid-error)

------------------------------------------------------------------
Your code has been rated at 0.00/10 (previous run: 0.00/10, +0.00)

[Pierre-Sassoulas]

We're going to fix this in 2.15.7, we're putting a new back-porting mechanism into place right now which temporarily block the release.

[XuehaiPan]

The pylint 2.15.7 was just released. But the fix is not included.

[Pierre-Sassoulas]

Yes, sorry it was hard to cherry-pick because it was based on other change that are exclusively on main (and I forgot to upgrade the milestones). The more likely scenario is that it's only going to be available in 2.16.0.