Merge pull request #65 from observerly/refactor/alembic/sqlite

refactor: Amended database usage to sqlite => v3.40.0.

.env.config

@@ -1,12 +1,6 @@
 FIRST_SUPERUSER_EMAIL
 FIRST_SUPERUSER_PASSWORD
 
-POSTGRES_USER="postgres"
-POSTGRES_PASSWORD="postgres"
-POSTGRES_SERVER="db"
-POSTGRES_PORT="5432"
-POSTGRES_DB="postgres"
-
 PROJECT_NAME="Perseus FastAPI"
 PROJECT_ENVIRONMENT
 SENTRY_DSN

.github/workflows/ci.yml

@@ -24,11 +24,6 @@ jobs:
       PROJECT_NAME: "Perseus FastAPI"
       FIRST_SUPERUSER_EMAIL: ${{ secrets.FIRST_SUPERUSER_EMAIL }}
       FIRST_SUPERUSER_PASSWORD: ${{ secrets.FIRST_SUPERUSER_PASSWORD }}
-      POSTGRES_USER: "postgres"
-      POSTGRES_PASSWORD: "postgres"
-      POSTGRES_SERVER: "db"
-      POSTGRES_PORT: "5432"
-      POSTGRES_DB: "postgres"
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       SERVER_NAME: "perseus"
       SERVER_HOST: "https://perseus.observerly.com"
@@ -43,11 +38,6 @@ jobs:
           echo PROJECT_NAME=${PROJECT_NAME} >> .env
           echo FIRST_SUPERUSER_EMAIL=${FIRST_SUPERUSER_EMAIL} >> .env
           echo FIRST_SUPERUSER_PASSWORD=${FIRST_SUPERUSER_PASSWORD} >> .env
-          echo POSTGRES_USER=${POSTGRES_USER} >> .env
-          echo POSTGRES_PASSWORD=${POSTGRES_PASSWORD} >> .env
-          echo POSTGRES_SERVER=${POSTGRES_SERVER} >> .env
-          echo POSTGRES_PORT=${POSTGRES_PORT} >> .env
-          echo POSTGRES_DB=${POSTGRES_DB} >> .env
           echo SENTRY_DSN=${SENTRY_DSN} >> .env
           echo SERVER_NAME=${SERVER_NAME} >> .env
           echo SERVER_HOST=${SERVER_HOST} >> .env

.gitignore

@@ -60,6 +60,8 @@ coverage.xml
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
+*.db.sqlite3
+*.db.sqlite3-journal
 
 # Flask stuff:
 instance/

alembic/env.py

@@ -1,6 +1,5 @@
 from __future__ import with_statement
 
-import os
 from logging.config import fileConfig
 
 from sqlalchemy import engine_from_config, pool
@@ -32,19 +31,7 @@
 
 
 def get_url():
-    user = os.getenv("POSTGRES_USER", "postgres")
-    password = os.getenv("POSTGRES_PASSWORD", "")
-    server = os.getenv("POSTGRES_SERVER", "db")
-    db = os.getenv("POSTGRES_DB", "app")
-    port = os.getenv("POSTGRES_PORT", "5432")
-    ssl = os.getenv("POSTGRES_SSL", "false")
-
-    url = f"postgresql://{user}:{password}@{server}:{port}/{db}"
-
-    if ssl == "true":
-        return f"{url}?sslmode=require"
-
-    return url
+    return "sqlite:///./perseus.db.sqlite3"
 
 
 def run_migrations_offline():

alembic/versions/d246e2f4a338_feat_added_body_model_e_g_star_galaxy_.py → alembic/versions/38650ea42efd_feat_added_body_model_e_g_star_galaxy_.py

@@ -1,17 +1,16 @@
 """feat: Added Body model (e.g., Star, Galaxy, Nebulae etc.)
 
-Revision ID: d246e2f4a338
+Revision ID: 38650ea42efd
 Revises: 
-Create Date: 2022-08-29 14:45:24.960549
+Create Date: 2022-11-29 17:31:11.612686
 
 """
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
 
 from alembic import op
 
 # revision identifiers, used by Alembic.
-revision = "d246e2f4a338"
+revision = "38650ea42efd"
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -21,7 +20,7 @@ def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table(
         "body",
-        sa.Column("uid", postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column("uid", sa.String(), nullable=False),
         sa.Column(
             "name", sa.String(length=180), nullable=True, comment="Common or IAU Name"
         ),
@@ -156,4 +155,4 @@ def downgrade():
     op.drop_index(op.f("ix_body_constellation"), table_name="body")
     op.drop_index(op.f("ix_body_bd"), table_name="body")
     op.drop_table("body")
-    # ### end Alembic commands ###root@504aaea6537f:/usr/src/app/alembic/versions#
+    # ### end Alembic commands ###

app/core/config.py

@@ -1,7 +1,7 @@
 from secrets import token_urlsafe
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import AnyHttpUrl, BaseSettings, EmailStr, HttpUrl, PostgresDsn, validator
+from pydantic import AnyHttpUrl, BaseSettings, EmailStr, HttpUrl, validator
 
 
 class Settings(BaseSettings):
@@ -43,26 +43,13 @@ def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str
             return v
         raise ValueError(v)
 
-    POSTGRES_SERVER: str
-    POSTGRES_USER: str
-    POSTGRES_PASSWORD: str
-    POSTGRES_DB: str
-    POSTGRES_PORT: str
-
-    SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None
+    SQLALCHEMY_DATABASE_URI: Optional[str] = None
 
     @validator("SQLALCHEMY_DATABASE_URI", pre=True)
     def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
         if isinstance(v, str):
             return v
-        return PostgresDsn.build(
-            scheme="postgresql",
-            user=values.get("POSTGRES_USER"),
-            password=values.get("POSTGRES_PASSWORD"),
-            host=values.get("POSTGRES_SERVER"),
-            port=values.get("POSTGRES_PORT"),
-            path=f"/{values.get('POSTGRES_DB') or ''}",
-        )
+        return "sqlite:///./perseus.db.sqlite3"
 
     SENTRY_DSN: Optional[HttpUrl]
 

app/crud/crud_body.py

@@ -117,7 +117,7 @@ def perform_name_search_filter(
 
         if name:
             query = query.filter(
-                or_(self.model.name.op("%")(name), self.model.iau.op("%")(name))
+                or_(self.model.name.op("LIKE")(name), self.model.iau.op("LIKE")(name))
             )
 
         return query
@@ -129,7 +129,7 @@ def perform_constellation_search_filter(
         constellation = getattr(query_params, "constellation", None)
 
         if constellation:
-            query = query.filter(self.model.constellation.op("%")(constellation))
+            query = query.filter(self.model.constellation.op("LIKE")(constellation))
 
         return query
 
@@ -167,9 +167,15 @@ def get_multi(
 
         count = query.count()
 
-        # Here we are ordering by apparent magnitude (mag) in descending order because
+        # Here we are ordering by apparent magnitude (mag) in ascending order because
         # negative magnitudes are actually "brighter" than positive magnitudes:
-        return query.order_by(self.model.m).offset(skip).limit(limit).all(), count
+        return (
+            query.order_by(func.coalesce(Body.m, 99999).asc())
+            .offset(skip)
+            .limit(limit)
+            .all(),
+            count,
+        )
 
 
 body = CRUDBody(Body)

app/db/session.py

@@ -1,16 +1,26 @@
-from sqlalchemy import create_engine
+import math
+
+from sqlalchemy import create_engine, event
 from sqlalchemy.orm import sessionmaker
 
 from app.core.config import settings
 
-if settings.PROJECT_ENVIRONMENT == "production":
-    if settings.SQLALCHEMY_DATABASE_URI is None:
-        raise Exception("settings.SQLALCHEMY_DATABASE_URI is None")
-    engine = create_engine(
-        "{}?sslmode=require".format(settings.SQLALCHEMY_DATABASE_URI),
-        pool_pre_ping=True,
-    )
-else:
-    engine = create_engine(settings.SQLALCHEMY_DATABASE_URI, pool_pre_ping=True)
+engine = create_engine(
+    settings.SQLALCHEMY_DATABASE_URI,
+    pool_pre_ping=True,
+    connect_args={"check_same_thread": False},
+)
+
+
+@event.listens_for(engine, "connect")
+def create_math_functions_on_connect(dbapi_connection, connection_record):
+    dbapi_connection.create_function("sin", 1, math.sin)
+    dbapi_connection.create_function("cos", 1, math.cos)
+    dbapi_connection.create_function("asin", 1, math.asin)
+    dbapi_connection.create_function("radians", 1, math.radians)
+    dbapi_connection.create_function("degrees", 1, math.degrees)
+    dbapi_connection.create_function("sqrt", 1, math.sqrt)
+    dbapi_connection.create_function("pow", 2, math.pow)
+
 
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

app/models/body.py

@@ -6,7 +6,6 @@
 from astropy.coordinates import EarthLocation, SkyCoord
 from astropy.time import Time
 from sqlalchemy import Column, Float, String, event
-from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.ext.hybrid import hybrid_method
 from sqlalchemy.sql import func
 
@@ -15,12 +14,16 @@
 FRAME = "icrs"
 
 
+def generate_uuid():
+    return str(uuid.uuid4())
+
+
 class Body(Base):
     # UID as primary key
     uid = Column(
-        UUID(as_uuid=True),
+        String,
         primary_key=True,
-        default=uuid.uuid4,
+        default=generate_uuid,
     )
 
     # Common Name

app/tests/api/api_v1/test_bodies.py

@@ -177,8 +177,8 @@ async def test_list_bodies_within_the_constellation_orion(client: AsyncClient) -
     assert body["results"][13]["name"] == "λ Orionis"
     assert body["results"][14]["name"] == "τ Orionis"
     assert body["results"][15]["name"] == "π⁴ Orionis"
-    assert body["results"][16]["name"] == "ζ Orionis"
-    assert body["results"][17]["name"] == "π⁵ Orionis"
+    assert body["results"][16]["name"] == "π⁵ Orionis"
+    assert body["results"][17]["name"] == "ζ Orionis"
     assert body["results"][18]["name"] == "σ Orionis"
     assert body["results"][19]["name"] == "ο² Orionis"
 

docker-compose.yml

@@ -10,7 +10,7 @@ services:
     ports:
       - 8001:5000
     volumes: 
-      - .:/app
+      - ./:/usr/src/app
     depends_on:
       - db
     links:
@@ -21,19 +21,6 @@ services:
     networks:
       - api
 
-  db:
-    image: postgres:14-alpine
-    container_name: postgres
-    volumes:
-      - postgres_data:/var/lib/postgresql/data/
-      - ./scripts/init_pgtrgm_extension.sql:/docker-entrypoint-initdb.d/init_pgtrgm_extension.sql
-    ports:
-      - 5432:5432
-    env_file:
-      - .env
-    networks:
-      - api
-
   redis:
     image: redis:alpine
     container_name: redis
@@ -44,9 +31,6 @@ services:
     networks:
       - api
 
-volumes:
-    postgres_data:
-
 networks:
   api:
     name: api

local.yml

@@ -48,11 +48,8 @@ services:
     ports:
       - 8001:5000
     volumes: 
-      - .:/app
-    depends_on:
-      - db
+      - ./:/usr/src/app
     links:
-      - db:db
       - redis:redis
     env_file:
       - .env
@@ -71,19 +68,6 @@ services:
       - web
       - api
 
-  db:
-    image: postgres:14-alpine
-    container_name: postgres
-    volumes:
-      - postgres_data:/var/lib/postgresql/data/
-      - ./scripts/init_pgtrgm_extension.sql:/docker-entrypoint-initdb.d/init_pgtrgm_extension.sql
-    ports:
-      - 5432:5432
-    env_file:
-      - .env
-    networks:
-      - api
-
   redis:
     image: redis:alpine
     container_name: redis
@@ -94,9 +78,6 @@ services:
     networks:
       - api
 
-volumes:
-    postgres_data:
-
 networks:
   web:
     name: web

scripts/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# Initialise the database
+python app/init_db_pre.py
+# Run Alembic Migrations
+alembic upgrade head
+# Seed the database
+python app/init_db_seed.py

scripts/init_db_pre.sh

@@ -1,2 +1,5 @@
 #!/bin/sh
+# This script is run before the database is initialized.
+rm -rf *.db.sqlite3
+# Initialise the database
 python app/init_db_pre.py