From 23ce8665a2967615747b4c26fde2cd230dbdd550 Mon Sep 17 00:00:00 2001 From: Peter Yefi Date: Wed, 7 Dec 2022 19:06:17 -0500 Subject: [PATCH] Added documentation for database setup --- .env | 16 +++++ DEPLOYMENT.md | 61 +++++++++++++++++++ exports/db_factory.py | 7 +-- exports/energy_systems/heat_pump_export.py | 2 +- imports/db_factory.py | 6 +- install_postgresql_linux.sh | 6 ++ persistence/__init__.py | 1 + persistence/base_repo.py | 19 +++--- persistence/db_config.py | 31 +++++----- persistence/db_setup.py | 11 ++++ persistence/repositories/city_repo.py | 6 +- .../repositories/heat_pump_simulation_repo.py | 8 +-- unittests/test_db_factory.py | 6 +- unittests/test_heat_pump_simulation.py | 6 +- 14 files changed, 140 insertions(+), 46 deletions(-) create mode 100644 .env create mode 100644 DEPLOYMENT.md create mode 100755 install_postgresql_linux.sh create mode 100644 persistence/db_setup.py diff --git a/.env b/.env new file mode 100644 index 00000000..82c1fada --- /dev/null +++ b/.env @@ -0,0 +1,16 @@ +# production database credentials +PROD_DB_USER=postgres +PROD_DB_PASSWORD= +PROD_DB_HOST=localhost +PROD_DB_PORT=5432 + +# test database credentials +TEST_DB_USER=postgres +TEST_DB_PASSWORD=postgres +TEST_DB_HOST=localhost +TEST_DB_PORT=5432 + +#Gitlab token +HUB_TOKEN=9s_CJYh5TcWhyYL416MM + +DEV_SECRET_NAME=dp.st.dev.Axvak1ILOlCOwUNGajv7fg5VPaacFR6OL1kdb3YGWHX \ No newline at end of file diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 00000000..7963ac64 --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,61 @@ +## Installing PostgreSQL Database Server on Linux (Ubuntu) ## +Execute the *install_postgresql_linux.sh* script to install PostgreSQL database +*NB: PostgreSQL DB Server runs on a default port of 5432.* + +## Installing PostgreSQL Database Server on Windows ## +1. Download a Windows installer from [this link](https://www.enterprisedb.com/downloads/postgres-postgresql-downloads). +2. Double click on the installer file and follow the prompts of the installation wizard +3. On the component selection page of the installation wizard make sure to select *PostgreSQL Server and Commandline tools* +4. You can optionally select pgAdmin 4 to install a graphical UI to access your database +5. On the password page when prompted, enter the default password (postgres) and confirm it +6. You can change the default password of 5432 on the port page. You should ensure that whatever port number you +provide is not used by another service. +7. Follow the installation wizard prompt to complete your installation. You can verify your installation by +searching for the *psql* tool from your start menu + +## Installing PostgreSQL Database Server on Mac OS X ## +1. Download the Mac OS X installer from [this link](https://www.enterprisedb.com/downloads/postgres-postgresql-downloads). +2. Launch the installation wizard by double-clicking it and follow the rest of steps as described above on +Installing PostgreSQL Database Server on Windows. + +NB: Hub has been tested with version 15 of PostgreSQL + +## Create Database and Database User ## +1. Connect to the PostgreSQL database server via psql by executing `sudo -u postgres psql`. You will be +be prompted for a password, the default password of *postgres* user is *postgres*. The above command may not work on +a Windows system using the default command line tool. You can access the psql tool from Windows start menu and follow +the rest of the instructions from step 2 below +2. Execute `create user with encrypted password '';` in the psql console to create a user. +3. Execute `create database ;` in the psql console to create a database. +4. Execute `grant all privileges on database to ;` to grant the all privileges on the database +to the user created in step 2 above. +5. The created database by default, has on schema named public which you can use. However, if you wish to create +another schema, you can do so by following [this link](https://www.postgresqltutorial.com/postgresql-administration/postgresql-create-schema/). + +**NB: You can grant selected privileges to the user on the database using commands [on this page](https://tableplus.com/blog/2018/04/postgresql-how-to-grant-access-to-users.html).* +The user should however have read and write permission to all tables in the database. You can as well create a database and user using the PgAdmin UI tool* + +## Setting Up Database Connection Parameters +1. Create a .env file that contains the configuration parameters as explained in the *Database Configuration Parameters* +section in persistence/README.md file. +2. The .env file should contain the following credentials: database user, database password, database host an,d database port +3. Provide the *absolute path* to the .env file to the persistence importers and exporters whenever using them in your code +as shown below: +```python +from exports.db_factory import DBFactory +from pathlib import Path + +dotenv_path = (Path(__file__).parent / '.env').resolve() +factory = DBFactory(db_name='hub_db', app_env='PROD', dotenv_path=dotenv_path, city=None) +``` + + +## Create Database Tables ## +Use the *DBSetUp* class in the persistence package to create the required database tables as described below +```python +from persistence import DBSetup +from pathlib import Path + +dotenv_path = (Path(__file__).parent / '.env').resolve() +DBSetup(db_name='hub_db', app_env='PROD', dotenv_path=dotenv_path) +``` \ No newline at end of file diff --git a/exports/db_factory.py b/exports/db_factory.py index c66c6ab7..f9fa401e 100644 --- a/exports/db_factory.py +++ b/exports/db_factory.py @@ -13,10 +13,9 @@ class DBFactory: DBFactory class """ - def __init__(self, city, db_name, app_env): - self._city = city - self._city_repo = CityRepo(db_name=db_name, app_env=app_env) - self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, app_env=app_env) + def __init__(self, db_name, app_env, dotenv_path): + self._city_repo = CityRepo(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) def get_city(self, city_id): """ diff --git a/exports/energy_systems/heat_pump_export.py b/exports/energy_systems/heat_pump_export.py index 17495835..a7e9e1df 100644 --- a/exports/energy_systems/heat_pump_export.py +++ b/exports/energy_systems/heat_pump_export.py @@ -56,7 +56,7 @@ class HeatPumpExport: insel_file_handler.write(insel_template) # Now run insel self._delete_existing_output_files() - os.system('insel {}'.format(insel_file)) + os.system('/usr/local/bin/insel {}'.format(insel_file)) # Writer headers to csv output files generated by insel self._write_insel_output_headers() # User output diff --git a/imports/db_factory.py b/imports/db_factory.py index 24baa426..532ce270 100644 --- a/imports/db_factory.py +++ b/imports/db_factory.py @@ -14,10 +14,10 @@ class DBFactory: DBFactory class """ - def __init__(self, city, db_name, app_env): + def __init__(self, city, db_name, dotenv_path, app_env): self._city = city - self._city_repo = CityRepo(db_name=db_name, app_env=app_env) - self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, app_env=app_env) + self._city_repo = CityRepo(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env) + self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env) def persist_city(self): """ diff --git a/install_postgresql_linux.sh b/install_postgresql_linux.sh new file mode 100755 index 00000000..70088df0 --- /dev/null +++ b/install_postgresql_linux.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +sudo apt-get update +sudo apt-get install postgresql \ No newline at end of file diff --git a/persistence/__init__.py b/persistence/__init__.py index ac736e74..c9d381ef 100644 --- a/persistence/__init__.py +++ b/persistence/__init__.py @@ -2,3 +2,4 @@ from .base_repo import BaseRepo from .repositories.city_repo import CityRepo from .repositories.building_repo import BuildingRepo from .repositories.heat_pump_simulation_repo import HeatPumpSimulationRepo +from .db_setup import DBSetup diff --git a/persistence/base_repo.py b/persistence/base_repo.py index 74c7e426..266fb4bf 100644 --- a/persistence/base_repo.py +++ b/persistence/base_repo.py @@ -12,17 +12,14 @@ from sqlalchemy.orm import Session class BaseRepo: - def __init__(self, db_name, app_env='TEST'): - self.config = BaseConfiguration(db_name, app_env) - self.engine = create_engine(self.config.conn_string()) - self.session = Session(self.engine) - - def __del__(self): - """ - Close database sessions - :return: - """ - self.session.close() + def __init__(self, db_name, dotenv_path: str, app_env='TEST'): + try: + self.config = BaseConfiguration(db_name, dotenv_path, app_env) + self.engine = create_engine(self.config.conn_string()) + self.session = Session(self.engine) + except ValueError as err: + print(f'Missing value for credentials: {err}') + diff --git a/persistence/db_config.py b/persistence/db_config.py index 9dbbac90..c23301bd 100644 --- a/persistence/db_config.py +++ b/persistence/db_config.py @@ -9,37 +9,40 @@ import os from dotenv import load_dotenv from sqlalchemy.ext.declarative import declarative_base - Base = declarative_base() -# load environmental variables -load_dotenv() - class BaseConfiguration(object): - """ + """ Base configuration class to hold common persistence configuration """ - def __init__(self, db_name: str, app_env='TEST'): - """ + + def __init__(self, db_name: str, dotenv_path: str, app_env='TEST'): + """ :param db_name: database name :param app_env: application environment, test or production + :param dotenv_path: the absolute path to dotenv file """ + try: + # load environmental variables + load_dotenv(dotenv_path=dotenv_path) self._db_name = db_name self._db_host = os.getenv(f'{app_env}_DB_HOST') self._db_user = os.getenv(f'{app_env}_DB_USER') self._db_pass = os.getenv(f'{app_env}_DB_PASSWORD') self._db_port = os.getenv(f'{app_env}_DB_PORT') self.hub_token = os.getenv('HUB_TOKEN') + except KeyError as err: + print(f'Error with credentials: {err}') - def conn_string(self): - """ + def conn_string(self): + """ Returns a connection string postgresql :return: connection string """ - if self._db_pass: - return f'postgresql://{self._db_user}:{self._db_pass}@{self._db_host}:{self._db_port}/{self._db_name}' - return f'postgresql://{self._db_user}@{self._db_host}:{self._db_port}/{self._db_name}' + if self._db_pass: + return f'postgresql://{self._db_user}:{self._db_pass}@{self._db_host}:{self._db_port}/{self._db_name}' + return f'postgresql://{self._db_user}@{self._db_host}:{self._db_port}/{self._db_name}' - def get_db_user(self): - return self._db_user + def get_db_user(self): + return self._db_user diff --git a/persistence/db_setup.py b/persistence/db_setup.py new file mode 100644 index 00000000..aa04ca9d --- /dev/null +++ b/persistence/db_setup.py @@ -0,0 +1,11 @@ +from persistence.models import City +from persistence import BaseRepo +from persistence.models import HeatPumpSimulation + + +class DBSetup: + + def __init__(self, db_name, app_env, dotenv_path): + repo = BaseRepo(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + City.__table__.create(bind=repo.engine, checkfirst=True) + HeatPumpSimulation.__table__.create(bind=repo.engine, checkfirst=True) \ No newline at end of file diff --git a/persistence/repositories/city_repo.py b/persistence/repositories/city_repo.py index db902270..8b359456 100644 --- a/persistence/repositories/city_repo.py +++ b/persistence/repositories/city_repo.py @@ -19,10 +19,10 @@ from typing import Union, Dict class CityRepo(BaseRepo): _instance = None - def __init__(self, db_name, app_env): - super().__init__(db_name, app_env) + def __init__(self, db_name: str, dotenv_path: str, app_env: str): + super().__init__(db_name, dotenv_path, app_env) - def __new__(cls, db_name, app_env): + def __new__(cls, db_name, dotenv_path, app_env): """ Implemented for a singleton pattern """ diff --git a/persistence/repositories/heat_pump_simulation_repo.py b/persistence/repositories/heat_pump_simulation_repo.py index 67950828..6f9e130f 100644 --- a/persistence/repositories/heat_pump_simulation_repo.py +++ b/persistence/repositories/heat_pump_simulation_repo.py @@ -15,11 +15,11 @@ from typing import Union, Dict class HeatPumpSimulationRepo(BaseRepo): _instance = None - def __init__(self, db_name, app_env): - super().__init__(db_name, app_env) - self._city_repo = CityRepo(db_name, app_env) + def __init__(self, db_name, dotenv_path, app_env): + super().__init__(db_name, dotenv_path, app_env) + self._city_repo = CityRepo(db_name, dotenv_path, app_env) - def __new__(cls, db_name, app_env): + def __new__(cls, db_name, dotenv_path, app_env): """ Implemented for a singleton pattern """ diff --git a/unittests/test_db_factory.py b/unittests/test_db_factory.py index c2870ef8..d97ef906 100644 --- a/unittests/test_db_factory.py +++ b/unittests/test_db_factory.py @@ -27,7 +27,7 @@ class TestDBFactory(TestCase): :return: None """ # Create test database - repo = BaseRepo(db_name='test_db', app_env='TEST') + repo = BaseRepo(db_name='test_db', app_env='TEST', dotenv_path='../.env') eng = create_engine(f'postgresql://{repo.config.get_db_user()}@/{repo.config.get_db_user()}') try: @@ -48,8 +48,8 @@ class TestDBFactory(TestCase): city_file = "../unittests/tests_data/C40_Final.gml" cls.city = GeometryFactory('citygml', city_file).city - cls._db_factory = DBFactory(city=cls.city, db_name='test_db', app_env='TEST') - cls._export_db_factory = ExportDBFactory(city=cls.city, db_name='test_db', app_env='TEST') + cls._db_factory = DBFactory(city=cls.city, db_name='test_db', app_env='TEST', dotenv_path='../.env') + cls._export_db_factory = ExportDBFactory(db_name='test_db', app_env='TEST', dotenv_path='../.env') def test_save_city(self): saved_city = self._db_factory.persist_city() diff --git a/unittests/test_heat_pump_simulation.py b/unittests/test_heat_pump_simulation.py index f67bb0be..de47d227 100644 --- a/unittests/test_heat_pump_simulation.py +++ b/unittests/test_heat_pump_simulation.py @@ -44,7 +44,7 @@ class TestHeatPumpSimulation(TestCase): Test setup :return: None """ - repo = BaseRepo(db_name='test_db', app_env='TEST') + repo = BaseRepo(db_name='test_db', app_env='TEST', dotenv_path='../.env') eng = create_engine(f'postgresql://{repo.config.get_db_user()}@/{repo.config.get_db_user()}') try: @@ -68,8 +68,8 @@ class TestHeatPumpSimulation(TestCase): cls._city = GeometryFactory('citygml', city_file).city EnergySystemsFactory('air source hp', cls._city).enrich() - cls._db_factory = DBFactory(city=cls._city, db_name='test_db', app_env='TEST') - cls._export_db_factory = ExportDBFactory(city=cls._city, db_name='test_db', app_env='TEST') + cls._db_factory = DBFactory(city=cls._city, db_name='test_db', app_env='TEST', dotenv_path='../.env') + cls._export_db_factory = ExportDBFactory(db_name='test_db', app_env='TEST', dotenv_path='../.env') def test_heat_pump_simulation_persistence(self): output = EnergySystemsExportFactory(city=self._city, user_input=hp_sim_data, hp_model='018',