Merge branch 'main' into auto-people

John Seekins · John Seekins · commit c30daa4a702b · 2022-07-11T10:41:45.000-06:00
diff --git a/.github/workflows/ca-docker.yml b/.github/workflows/ca-docker.yml
@@ -9,9 +9,10 @@ on:
       - "poetry.lock"
       - "Dockerfile.california"
       - "scrapers/ca/*"
-jobs: 
+jobs:
   publish:
-    steps: 
+    runs-on: ubuntu-latest
+    steps:
       - uses: actions/checkout@v2
       - name: build california docker image
         uses: docker/build-push-action@v1.1.0
@@ -22,4 +23,3 @@ jobs:
           dockerfile: /github/workspace/Dockerfile.california
           tag_with_ref: true
           tags: latest
-    runs-on: ubuntu-18.04
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -4,9 +4,10 @@ on:
     branches: [ main ]
     tags:
       - '*'
-jobs: 
+jobs:
   publish:
-    steps: 
+    runs-on: ubuntu-latest
+    steps:
       - uses: actions/checkout@v2
       - name: build docker image
         uses: docker/build-push-action@v1.1.0
@@ -17,4 +18,3 @@ jobs:
           dockerfile: /github/workspace/Dockerfile
           tag_with_ref: true
           tags: latest
-    runs-on: ubuntu-18.04
diff --git a/.gitignore b/.gitignore
@@ -39,6 +39,7 @@ venv
 
 # Binary data files and dumps, from Open States and state websites
 openstates.sqlite3
+openstates-postgres/
 dump/
 tmp.*
 *.mdb
diff --git a/Dockerfile b/Dockerfile
@@ -1,37 +1,53 @@
 FROM python:3.9-slim
 LABEL maintainer="James Turk <dev@jamesturk.net>"
 
-ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONIOENCODING='utf-8' LANG='C.UTF-8'
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONIOENCODING='utf-8'
+ENV LANG='C.UTF-8'
 
-RUN apt update && apt install -y --no-install-recommends \
-      git \
-      build-essential \
+RUN apt-get update -qq \
+    && apt-get install -y -qq --no-install-recommends \
       curl \
+      wget \
       unzip \
+      mdbtools \
+      libpq5 \
+      libgdal28 \
+      build-essential \
+      git \
       libssl-dev \
       libffi-dev \
       freetds-dev \
-      python3-virtualenv \
       libxml2-dev \
       libxslt-dev \
       libyaml-dev \
       poppler-utils \
       libpq-dev \
       libgdal-dev \
       libgeos-dev \
-      wget \
-      unzip \
-#     libcrypto1.1 \
-      mdbtools && \
-      rm -rf /var/lib/apt/lists/*
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-ADD . /opt/openstates/openstates
+ADD poetry.lock /opt/openstates/openstates/
+ADD pyproject.toml /opt/openstates/openstates/
 WORKDIR /opt/openstates/openstates/
 ENV PYTHONPATH=./scrapers
 
-RUN set -ex \
-    && pip install poetry \
-    && poetry install
+# the last step cleans out temporarily downloaded artifacts for poetry, shrinking our build
+RUN pip --no-cache-dir --disable-pip-version-check install poetry \
+    && poetry install --no-root \
+    && apt-get remove -y -qq \
+      build-essential \
+      git \
+      libpq-dev \
+    && apt-get autoremove -y -qq \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD . /opt/openstates/openstates/
+RUN poetry install \
+    && rm -r /root/.cache/pypoetry/cache /root/.cache/pypoetry/artifacts/
 
 ENV OPENSSL_CONF=/opt/openstates/openstates/openssl.cnf
 
diff --git a/Dockerfile.california b/Dockerfile.california
@@ -1,40 +1,55 @@
 FROM python:3.9-slim
 LABEL maintainer="James Turk <dev@jamesturk.net>"
 
-ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONIOENCODING='utf-8' LANG='C.UTF-8'
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONIOENCODING='utf-8'
+ENV LANG='C.UTF-8'
 
-RUN apt update && apt install -y --no-install-recommends \
-      git \
-      build-essential \
+RUN apt-get update -qq \
+    && apt-get install -y -qq --no-install-recommends \
       curl \
+      wget \
       unzip \
+      mdbtools \
+      libpq5 \
+      libgdal28 \
+      libmariadb-dev \
+      mariadb-server \
+      mariadb-client \
+      build-essential \
+      git \
       libssl-dev \
       libffi-dev \
       freetds-dev \
-      python3-virtualenv \
       libxml2-dev \
       libxslt-dev \
       libyaml-dev \
       poppler-utils \
       libpq-dev \
       libgdal-dev \
       libgeos-dev \
-      wget \
-      unzip \
-#     libcrypto1.1 \
-      libmariadb-dev \
-      mariadb-server \
-      mariadb-client \
-      mdbtools && \
-      rm -rf /var/lib/apt/lists/*
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-ADD . /opt/openstates/openstates
+ADD poetry.lock /opt/openstates/openstates/
+ADD pyproject.toml /opt/openstates/openstates/
 WORKDIR /opt/openstates/openstates/
 ENV PYTHONPATH=./scrapers
 
-RUN set -ex \
-    && pip install poetry \
-    && poetry install --extras "california"
+# the last step cleans out temporarily downloaded artifacts for poetry, shrinking our build
+RUN pip --no-cache-dir --disable-pip-version-check install poetry \
+    && poetry install \
+    && rm -r /root/.cache/pypoetry/cache /root/.cache/pypoetry/artifacts/ \
+    && apt-get remove -y -qq \
+      build-essential \
+      git \
+      libpq-dev \
+    && apt-get autoremove -y -qq \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD . /opt/openstates/openstates/
 
 ENV OPENSSL_CONF=/opt/openstates/openstates/openssl.cnf
 
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,11 +4,6 @@ networks:
   openstates-network:
       name: openstates-network
 
-volumes:
-  openstates-postgres:
-    name: openstates-postgres
-    external: true
-
 services:
   scrape:
     build: .
@@ -22,30 +17,30 @@ services:
     - VIRGINIA_FTP_USER
     - VIRGINIA_FTP_PASSWORD
     volumes:
-    - .:/opt/openstates/openstates/
+      - .:/opt/openstates/openstates/
     entrypoint: ["poetry", "run", "os-update"]
     networks:
-    - openstates-network
+      - openstates-network
   mysql:
     image: mariadb:10.3
 #    command: mysqld_safe --max_allowed_packet=512M
     ports:
-    - "3306:3306"
+      - "3306:3306"
     environment:
-    - MYSQL_DATABASE=capublic
-    - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+      - MYSQL_DATABASE=capublic
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
     networks:
-    - openstates-network
+      - openstates-network
   ca-scrape:
     build:
       context: .
       dockerfile: Dockerfile.california
     environment:
-    - MYSQL_HOST=mysql
-    - DATABASE_URL=postgres://openstates:openstates@db/openstatesorg
-    - PYTHONPATH=./scrapers
+      - MYSQL_HOST=mysql
+      - DATABASE_URL=postgres://openstates:openstates@db/openstatesorg
+      - PYTHONPATH=./scrapers
     volumes:
-    - .:/opt/openstates/openstates/
+      - .:/opt/openstates/openstates/
     entrypoint: ["poetry", "run", "os-update"]
     networks:
     - openstates-network
@@ -55,13 +50,13 @@ services:
       dockerfile: Dockerfile.california
     entrypoint: /opt/openstates/openstates/scrapers/ca/download.sh
     environment:
-    - MYSQL_HOST=mysql
+      - MYSQL_HOST=mysql
     volumes:
-    - .:/opt/openstates/openstates/
+      - .:/opt/openstates/openstates/
     depends_on:
-    - mysql
+      - mysql
     networks:
-    - openstates-network
+      - openstates-network
   db:
     image: "mdillon/postgis:11-alpine"
     hostname: "db"
@@ -72,6 +67,6 @@ services:
       POSTGRES_USER: openstates
       POSTGRES_DB: openstatesorg
     volumes:
-      - openstates-postgres:/var/lib/postgresql/data
+      - ./openstates-postgres/:/var/lib/postgresql/data
     networks:
       - openstates-network
diff --git a/scrapers/utils/__init__.py b/scrapers/utils/__init__.py
@@ -3,13 +3,15 @@
 from .lxmlize import LXMLMixin  # noqa
 from .lxmlize import url_xpath  # noqa
 
+_phone_pattern = re.compile(r"\(?\d{3}\)?\s?-?\d{3}-?\d{4}")
+_email_pattern = re.compile(r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\." r"[a-zA-Z]{2,}\b")
+
 
 def validate_phone_number(phone_number):
     is_valid = False
 
     # Phone format validation regex.
-    phone_pattern = re.compile(r"\(?\d{3}\)?\s?-?\d{3}-?\d{4}")
-    phone_match = phone_pattern.match(phone_number)
+    phone_match = _phone_pattern.match(phone_number)
     if phone_match is not None:
         is_valid = True
 
@@ -19,10 +21,7 @@ def validate_phone_number(phone_number):
 def validate_email_address(email_address):
     is_valid = False
 
-    email_pattern = re.compile(
-        r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\." r"[a-zA-Z]{2,}\b"
-    )
-    email_match = email_pattern.match(email_address)
+    email_match = _email_pattern.match(email_address)
     if email_match is not None:
         is_valid = True
 
diff --git a/scrapers/utils/actions.py b/scrapers/utils/actions.py
@@ -29,8 +29,8 @@ def __new__(
         for regex in regexes:
             if isinstance(regex, string_types):
                 if flexible_whitespace:
-                    regex = re.sub(r"\s{1,4}", r"\\s{,10}", regex)
-                compiled_regexes.append(re.compile(regex))
+                    c_regex = re.sub(r"\s{1,4}", r"\\s{,10}", regex)
+                compiled_regexes.append(re.compile(c_regex))
             else:
                 compiled_regexes.append(regex)
 
diff --git a/scripts/init-db.sh b/scripts/init-db.sh
@@ -5,7 +5,6 @@ unset DATABASE_URL
 
 # stop database and remove volume
 docker-compose down
-docker volume rm openstates-postgres || true
 docker-compose up -d db
 sleep 3
 
