Merge pull request #1104 from cmu-delphi/rzatserkovnyi/common-package

Add support for src/common directory

Author: korlaxxalrok

.github/workflows/ci.yaml

@@ -57,11 +57,13 @@ jobs:
           docker build -t delphi_web_epidata -f ./devops/Dockerfile .
           cd ../../../
 
+      # MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
+      # see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
       - name: Start services
         run: |
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
-          docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
+          docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
           docker ps
 
       - run: |

deploy.json

@@ -17,6 +17,15 @@
       "add-header-comment": true
     },
 
+    "// common",
+    {
+      "type": "move",
+      "src": "src/common/",
+      "dst": "[[package]]/common/",
+      "match": "^.*\\.(py)$",
+      "add-header-comment": true
+    },
+
     "// server",
     {
       "type": "move",
@@ -47,15 +56,6 @@
       "add-header-comment": true
     },
 
-    "// acquisition - common",
-    {
-      "type": "move",
-      "src": "src/acquisition/common/",
-      "dst": "[[package]]/acquisition/common/",
-      "match": "^.*\\.(py)$",
-      "add-header-comment": true
-    },
-
     "// acquisition - fluview",
     {
       "type": "move",

dev/local/Makefile

@@ -91,8 +91,11 @@ web:
 		cd -
 
 	@# Run the web server
+	@# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run.
+	@# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name
 	@docker run --rm -p 127.0.0.1:10080:80 \
 		$(M1) \
+		--env "MODULE_NAME=delphi.epidata.server.main" \
 		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
 		--network delphi-net --name delphi_web_epidata \

devops/Dockerfile

@@ -8,14 +8,16 @@ LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epida
 
 COPY ./devops/gunicorn_conf.py /app
 COPY ./devops/start_wrapper.sh /
-COPY ./src/server/ /app/app/
-COPY --from=builder ./src/build/lib/ /app/app/lib/
+RUN mkdir -p /app/delphi/epidata
+COPY ./src/server /app/delphi/epidata/server
+COPY ./src/common /app/delphi/epidata/common
+COPY --from=builder ./src/build/lib/ /app/delphi/epidata/lib/
 
 COPY requirements.api.txt /app/requirements_also.txt
 
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime \
-      && rm -rf /app/app/__pycache__ /app/app/*.php \
-      && chmod -R o+r /app/app \
+      && rm -rf /app/delphi/epidata/__pycache__ \
+      && chmod -R o+r /app/delphi/epidata \
       && chmod 755 /start_wrapper.sh \
       && pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
 # the file /tmp/requirements.txt is created in the parent docker definition.  (see:

src/acquisition/common/logger.py

@@ -11,7 +11,7 @@
 
 # first party
 import delphi.operations.secrets as secrets
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 Columndef = namedtuple("Columndef", "csv_name sql_name dtype")
 

src/acquisition/covid_hosp/common/database.py

@@ -7,7 +7,7 @@
 
 # first party
 from delphi.epidata.acquisition.covidcast.database import Database
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 from delphi.epidata.client.delphi_epidata import Epidata
 
 def get_argument_parser():

src/acquisition/covidcast/covidcast_meta_cache_updater.py

@@ -15,8 +15,8 @@
 # first party
 from delphi_utils import Nans
 from delphi.utils.epiweek import delta_epiweeks
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.covidcast_row import CovidcastRow
+from delphi.epidata.common.logger import get_structured_logger
 
 DataFrameRow = NamedTuple('DFRow', [
   ('geo_id', str),

src/acquisition/covidcast/csv_importer.py

@@ -11,7 +11,7 @@
 from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter, PathDetails
 from delphi.epidata.acquisition.covidcast.database import Database, DBLoadStateException
 from delphi.epidata.acquisition.covidcast.file_archiver import FileArchiver
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 def get_argument_parser():

src/acquisition/covidcast/csv_to_database.py

@@ -14,8 +14,8 @@
 
 # first party
 import delphi.operations.secrets as secrets
-from delphi.epidata.acquisition.common.logger import get_structured_logger
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
+from delphi.epidata.common.logger import get_structured_logger
+from delphi.epidata.common.covidcast_row import CovidcastRow
 
 
 class DBLoadStateException(Exception):

src/acquisition/covidcast/database.py

@@ -8,7 +8,7 @@
 
 # first party
 from delphi.epidata.acquisition.covidcast.database import Database
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 def get_argument_parser():

src/acquisition/covidcast/delete_batch.py

@@ -6,7 +6,7 @@
 import shutil
 
 # first party
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 class FileArchiver:
   """Archives files by moving and compressing."""

src/acquisition/covidcast/file_archiver.py

@@ -15,7 +15,7 @@
 # first party
 import covidcast
 import delphi.operations.secrets as secrets
-from delphi.epidata.acquisition.common.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 LOOKBACK_DAYS_FOR_COVERAGE = 56

src/acquisition/covidcast/signal_dash_data_generator.py

@@ -6,7 +6,7 @@
 import pandas as pd
 
 from delphi_utils import Nans
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow
+from delphi.epidata.common.covidcast_row import CovidcastRow
 from delphi.epidata.acquisition.covidcast.database import Database
 from delphi.epidata.server.utils.dates import day_to_time_value, time_value_to_day
 import delphi.operations.secrets as secrets

src/acquisition/covidcast/test_utils.py

@@ -6,7 +6,7 @@
 from sqlalchemy.engine import Connection, Engine
 from werkzeug.local import LocalProxy
 
-from .utils.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 from ._config import SECRET, SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS
 from ._exceptions import DatabaseErrorException, EpiDataException
 

src/acquisition/covidcast/covidcast_row.py renamed to src/common/covidcast_row.py

@@ -8,7 +8,7 @@
 
 from ._config import MAX_RESULTS, MAX_COMPATIBILITY_RESULTS
 from ._common import is_compatibility_mode
-from .utils.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 
 def print_non_standard(format: str, data):

src/server/utils/logger.py renamed to src/common/logger.py

@@ -8,7 +8,7 @@
 from .._params import extract_strings
 from .._printer import create_printer
 from .._query import filter_fields
-from ..utils.logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 bp = Blueprint("covidcast_meta", __name__)
 

src/server/_common.py

@@ -10,7 +10,7 @@
 from epiweeks import Week, Year
 from typing_extensions import TypeAlias
 
-from .logger import get_structured_logger
+from delphi.epidata.common.logger import get_structured_logger
 
 # Alias for a sequence of date ranges (int, int) or date integers
 IntRange: TypeAlias = Union[Tuple[int, int], int]

src/server/_printer.py

@@ -4,7 +4,7 @@
 from pandas.testing import assert_frame_equal
 
 from delphi_utils.nancodes import Nans
-from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow, set_df_dtypes
+from delphi.epidata.common.covidcast_row import CovidcastRow, set_df_dtypes
 from delphi.epidata.acquisition.covidcast.test_utils import (
     CovidcastTestRow,
     covidcast_rows_as_api_compatibility_row_df, 
@@ -14,7 +14,7 @@
 )
 
 # py3tester coverage target (equivalent to `import *`)
-__test_target__ = 'delphi.epidata.acquisition.covidcast.covidcast_row'
+__test_target__ = 'delphi.epidata.common.covidcast_row'
 
 
 class TestCovidcastRows(unittest.TestCase):