Handle joint responsible committee

See https://www.europarl.europa.eu/doceo/document/RULES-9-2022-07-11-RULE-058_EN.html

Author: tillprochaska

backend/howtheyvote/alembic/versions/8d1995cb0bed_make_responsible_committee_column_json_.py

@@ -0,0 +1,26 @@
+"""Make responsible_committee column JSON in votes table
+
+Revision ID: 8d1995cb0bed
+Revises: 848ef24718dd
+Create Date: 2025-03-16 17:13:08.000602
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "8d1995cb0bed"
+down_revision = "848ef24718dd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_column("votes", "responsible_committee")
+    op.add_column("votes", sa.Column("responsible_committees", sa.JSON))
+
+
+def downgrade() -> None:
+    op.drop_column("votes", "responsible_committees")
+    op.add_column("votes", sa.Column("responsible_committee", sa.Unicode))

backend/howtheyvote/api/serializers.py

@@ -258,16 +258,16 @@ class BaseVoteDict(TypedDict):
     """Concepts from the [EuroVoc](https://eur-lex.europa.eu/browse/eurovoc.html) thesaurus
     that are related to this vote"""
 
-    responsible_committee: CommitteeDict | None
+    responsible_committees: list[CommitteeDict] | None
     """Committee responsible for the legislative procedure"""
 
 
 def serialize_base_vote(vote: Vote) -> BaseVoteDict:
     geo_areas = [serialize_country(geo_area) for geo_area in vote.geo_areas]
     eurovoc_concepts = [serialize_eurovoc_concept(ec) for ec in vote.eurovoc_concepts]
-    responsible_committee = (
-        serialize_committee(vote.responsible_committee) if vote.responsible_committee else None
-    )
+    responsible_committees = [
+        serialize_committee(committee) for committee in vote.responsible_committees
+    ]
 
     return {
         "id": vote.id,
@@ -278,7 +278,7 @@ def serialize_base_vote(vote: Vote) -> BaseVoteDict:
         "is_featured": vote.is_featured,
         "geo_areas": geo_areas,
         "eurovoc_concepts": eurovoc_concepts,
-        "responsible_committee": responsible_committee,
+        "responsible_committees": responsible_committees,
     }
 
 

backend/howtheyvote/export/__init__.py

@@ -4,11 +4,11 @@
 import tempfile
 from typing import Any, TypedDict
 
-from sqlalchemy import select
+from sqlalchemy import func, select
 from structlog import get_logger
 
 from ..db import Session
-from ..models import Member, Vote
+from ..models import Committee, Member, Vote
 from ..vote_stats import count_vote_positions
 from .csvw_helpers import Table
 
@@ -144,9 +144,6 @@ class VoteRow(TypedDict):
     procedure_title: str | None
     """Title of the legislative procedure as listed in the Legislative Observatory"""
 
-    responsible_committee_code: str | None
-    """Committee responsible for the legislative procedure"""
-
     count_for: int
     """Number of MEPs who voted in favor"""
 
@@ -182,6 +179,29 @@ class MemberVoteRow(TypedDict):
     of the vote. This is not necessarily the MEP’s current political group."""
 
 
+class CommitteeRow(TypedDict):
+    """Each row represents a committee of the European Parliament."""
+
+    code: str
+    """Unique identifier of the committee"""
+
+    label: str
+    """Label"""
+
+    abbreviation: str
+    """Abbreviation"""
+
+
+class ResponsibleCommitteeVotes(TypedDict):
+    """Committee responsible for the legislative procedure a vote is part of."""
+
+    vote_id: int
+    """Vote ID"""
+
+    committee_code: str
+    """Committee code"""
+
+
 class Export:
     def __init__(self, outdir: pathlib.Path):
         self.outdir = outdir
@@ -228,6 +248,20 @@ def __init__(self, outdir: pathlib.Path):
             primary_key=["member_id", "vote_id"],
         )
 
+        self.committees = Table(
+            row_type=CommitteeRow,
+            outdir=self.outdir,
+            name="committees",
+            primary_key=["code"],
+        )
+
+        self.responsible_committee_votes = Table(
+            row_type=ResponsibleCommitteeVotes,
+            outdir=self.outdir,
+            name="responsible_committee_votes",
+            primary_key=["vote_id", "committee_code"],
+        )
+
     def run(self) -> None:
         self.fetch_members()
         self.write_export_timestamp()
@@ -239,10 +273,13 @@ def run(self) -> None:
                 self.group_memberships,
                 self.votes,
                 self.member_votes,
+                self.committees,
+                self.responsible_committee_votes,
             ]
         )
         self.export_members()
         self.export_votes()
+        self.export_committees()
 
     def fetch_members(self) -> None:
         self.members_by_id: dict[int, Member] = {}
@@ -351,10 +388,6 @@ def export_votes(self) -> None:
                 if idx % 1000 == 0:
                     log.info("Writing vote", index=idx)
 
-                responsible_committee_code = (
-                    vote.responsible_committee.code if vote.responsible_committee else None
-                )
-
                 position_counts = count_vote_positions(vote.member_votes)
 
                 votes.write_row(
@@ -368,7 +401,6 @@ def export_votes(self) -> None:
                         "is_featured": vote.is_featured,
                         "procedure_reference": vote.procedure_reference,
                         "procedure_title": vote.procedure_title,
-                        "responsible_committee_code": responsible_committee_code,
                         "count_for": position_counts["FOR"],
                         "count_against": position_counts["AGAINST"],
                         "count_abstention": position_counts["ABSTENTION"],
@@ -392,6 +424,30 @@ def export_votes(self) -> None:
                         }
                     )
 
+    def export_committees(self) -> None:
+        log.info("Exporting committees")
+
+        with self.committees.open() as committees:
+            exp = func.json_each(Vote.responsible_committees).table_valued("value")
+            query = (
+                select(func.distinct(exp.c.value)).select_from(Vote, exp).order_by(exp.c.value)
+            )
+            committee_codes = Session.execute(query).scalars()
+
+            for committee_code in committee_codes:
+                # `if True else None` is a hack to make mypy treat this as a normal value
+                # expression and not as a type expression. If this keeps causing type checking
+                # issues we might want to reconsider the use of metaclasses for this purpose.
+                # See: https://github.com/python/mypy/issues/15107
+                committee = Committee[committee_code] if True else None
+                committees.write_row(
+                    {
+                        "code": committee.code,
+                        "label": committee.label,
+                        "abbreviation": committee.abbreviation,
+                    }
+                )
+
 
 def generate_export(path: pathlib.Path) -> None:
     with tempfile.TemporaryDirectory() as outdir:

backend/howtheyvote/models/vote.py

@@ -97,7 +97,7 @@ class Vote(BaseWithId):
     eurovoc_concepts: Mapped[list[EurovocConcept]] = mapped_column(
         ListType(EurovocConceptType())
     )
-    responsible_committee: Mapped[Committee] = mapped_column(CommitteeType())
+    responsible_committees: Mapped[list[Committee]] = mapped_column(ListType(CommitteeType()))
     press_release: Mapped[str | None] = mapped_column(sa.Unicode)
     issues: Mapped[list[DataIssue]] = mapped_column(ListType(sa.Enum(DataIssue)))
 

backend/howtheyvote/scrapers/votes.py

@@ -412,7 +412,7 @@ def _url(self) -> str:
     def _extract_data(self, doc: BeautifulSoup) -> Fragment:
         title = self._title(doc)
         geo_areas = self._geo_areas(doc)
-        responsible_committee = self._responsible_committee(doc)
+        responsible_committees = self._responsible_committees(doc)
         self._log.info(
             "Extracted procedure information",
             title=title,
@@ -426,7 +426,7 @@ def _extract_data(self, doc: BeautifulSoup) -> Fragment:
             data={
                 "procedure_title": title,
                 "geo_areas": geo_areas,
-                "responsible_committee": responsible_committee,
+                "responsible_committees": responsible_committees,
             },
         )
 
@@ -471,30 +471,30 @@ def _geo_areas(self, doc: BeautifulSoup) -> list[str]:
 
         return geo_areas
 
-    def _responsible_committee(self, doc: BeautifulSoup) -> str | None:
+    def _responsible_committees(self, doc: BeautifulSoup) -> list[str]:
         table = doc.select_one(
-            '#erpl_accordion-committee table:has(th:-soup-contains("Committee responsible"))'
+            "#erpl_accordion-committee :where("
+            + 'table:has(th:-soup-contains("Committee responsible")),'
+            + 'table:has(th:-soup-contains("Joint committee responsible"))'
+            + ")"
         )
 
         if not table:
             return None
 
-        if len(table.select("tbody tr")) > 1:
-            # We assume that there is at most one responsible committee
-            log.warning("More than one responsible committee found")
+        badges = table.select("tbody tr .erpl_badge-committee")
+        committees = set()
 
-        badge = table.select_one("tbody tr .erpl_badge-committee")
+        for badge in badges:
+            text = badge.text.strip()
+            committee = Committee.get(text)
 
-        if not badge:
-            return None
-
-        text = badge.text.strip()
-        committee = Committee.get(text)
+            if not committee:
+                raise ScrapingError(f"Could not find committee {text}")
 
-        if not committee:
-            raise ScrapingError(f"Could not find committee {text}")
+            committees.add(committee.code)
 
-        return committee.code
+        return committees
 
 
 class EurlexProcedureScraper(BeautifulSoupScraper):

backend/howtheyvote/store/mappings.py

@@ -60,7 +60,9 @@ def map_vote(record: CompositeRecord) -> Vote:
     member_votes = [deserialize_member_vote(mv) for mv in record.first("member_votes")]
     geo_areas = {Country[code] for code in record.chain("geo_areas")}
     eurovoc_concepts = {EurovocConcept[id_] for id_ in record.chain("eurovoc_concepts")}
-    responsible_committee = Committee.get(record.first("responsible_committee"))
+    responsible_committees = {
+        Committee[code] for code in record.chain("responsible_committees")
+    }
 
     press_release = record.first("press_release")
     is_featured = record.first("is_featured") or press_release is not None
@@ -82,7 +84,7 @@ def map_vote(record: CompositeRecord) -> Vote:
         member_votes=member_votes,
         geo_areas=geo_areas,
         eurovoc_concepts=eurovoc_concepts,
-        responsible_committee=responsible_committee,
+        responsible_committees=responsible_committees,
         press_release=press_release,
         issues=record.chain("issues"),
     )

backend/tests/api/test_votes_api.py

@@ -536,7 +536,7 @@ def test_votes_api_show(records, db_session, api):
         ],
         "geo_areas": [],
         "eurovoc_concepts": [],
-        "responsible_committee": None,
+        "responsible_committees": [],
         "related": [],
         "sources": [
             {

backend/tests/export/test_init.py

@@ -5,6 +5,7 @@
 
 from howtheyvote.export import Export
 from howtheyvote.models import (
+    Committee,
     Country,
     Group,
     GroupMembership,
@@ -126,6 +127,7 @@ def test_export_votes(db_session, tmp_path):
                 position=VotePosition.FOR,
             ),
         ],
+        responsible_committees=[Committee["AFET"]],
     )
 
     db_session.add_all([member, vote])
@@ -138,8 +140,8 @@ def test_export_votes(db_session, tmp_path):
     votes_meta = tmp_path.joinpath("votes.csv-metadata.json")
 
     expected = (
-        "id,timestamp,display_title,reference,description,is_main,is_featured,procedure_reference,procedure_title,responsible_committee_code,count_for,count_against,count_abstention,count_did_not_vote\n"
-        "123456,2024-01-01 00:00:00,Lorem Ipsum,,,False,False,,,,1,0,0,0\n"
+        "id,timestamp,display_title,reference,description,is_main,is_featured,procedure_reference,procedure_title,count_for,count_against,count_abstention,count_did_not_vote\n"
+        "123456,2024-01-01 00:00:00,Lorem Ipsum,,,False,False,,,1,0,0,0\n"
     )
 
     assert votes_csv.read_text() == expected
@@ -153,6 +155,14 @@ def test_export_votes(db_session, tmp_path):
     assert member_votes_csv.read_text() == expected
     assert member_votes_meta.is_file()
 
+    committees_csv = tmp_path.joinpath("committees.csv")
+    committees_meta = tmp_path.joinpath("committees.csv-metadata.json")
+
+    expected = "code,label,abbreviation\nAFET,Committee on Foreign Affairs,AFET\n"
+
+    assert committees_csv.read_text() == expected
+    assert committees_meta.is_file()
+
 
 def test_export_votes_country_group(db_session, tmp_path):
     member = Member(