Added documentation for database setup

.env

@@ -0,0 +1,16 @@
+# production database credentials
+PROD_DB_USER=postgres
+PROD_DB_PASSWORD=
+PROD_DB_HOST=localhost
+PROD_DB_PORT=5432
+
+# test database credentials
+TEST_DB_USER=postgres
+TEST_DB_PASSWORD=postgres
+TEST_DB_HOST=localhost
+TEST_DB_PORT=5432
+
+#Gitlab token
+HUB_TOKEN=9s_CJYh5TcWhyYL416MM
+
+DEV_SECRET_NAME=dp.st.dev.Axvak1ILOlCOwUNGajv7fg5VPaacFR6OL1kdb3YGWHX

DEPLOYMENT.md

@@ -0,0 +1,61 @@
+## Installing PostgreSQL Database Server on Linux (Ubuntu) ##
+Execute the *install_postgresql_linux.sh* script to install PostgreSQL database
+*NB: PostgreSQL DB Server runs on a default port of 5432.*
+
+## Installing PostgreSQL Database Server on Windows ##
+1. Download a Windows installer from [this link](https://www.enterprisedb.com/downloads/postgres-postgresql-downloads).
+2. Double click on the installer file and follow the prompts of the installation wizard
+3. On the component selection page of the installation wizard make sure to select *PostgreSQL Server and Commandline tools*
+4. You can optionally select pgAdmin 4 to install a graphical UI to access your database
+5. On the password page when prompted, enter the default password (postgres) and confirm it
+6. You can change the default password of 5432 on the port page. You should ensure that whatever port number you
+provide is not used by another service.
+7. Follow the installation wizard prompt to complete your installation. You can verify your installation by
+searching for the *psql* tool from your start menu
+
+## Installing PostgreSQL Database Server on Mac OS X ##
+1. Download the Mac OS X installer from [this link](https://www.enterprisedb.com/downloads/postgres-postgresql-downloads).
+2. Launch the installation wizard by double-clicking it and follow the rest of steps as described above on 
+Installing PostgreSQL Database Server on Windows.
+
+NB: Hub has been tested with version 15 of PostgreSQL
+
+## Create Database and Database User ##
+1. Connect to the PostgreSQL database server via psql by executing `sudo -u postgres psql`. You will be
+be prompted for a password, the default password of *postgres* user is *postgres*. The above command may not work on 
+a Windows system using the default command line tool. You can access the psql tool from Windows start menu and follow 
+the rest of the instructions from step 2 below
+2. Execute `create user <username> with encrypted password '<password>';` in the psql console to create a user.
+3. Execute `create database <database-name>;` in the psql console to create a database.
+4. Execute `grant all privileges on database <database-name> to <username>;` to grant the all privileges on the database
+to the user created in step 2 above. 
+5. The created database by default, has on schema named public which you can use. However, if you wish to create
+another schema, you can do so by following [this link](https://www.postgresqltutorial.com/postgresql-administration/postgresql-create-schema/).
+
+**NB: You can grant selected privileges to the user on the database using commands  [on this page](https://tableplus.com/blog/2018/04/postgresql-how-to-grant-access-to-users.html).*
+The user should however have read and write permission to all tables in the database. You can as well create a database and user using the PgAdmin UI tool*
+
+## Setting Up Database Connection Parameters
+1. Create a .env file that contains the configuration parameters as explained in the *Database Configuration Parameters*
+section in persistence/README.md file.
+2. The .env file should contain the following credentials: database user, database password, database host an,d database port
+3. Provide the *absolute path* to the .env file to the persistence importers and exporters whenever using them in your code
+as shown below:
+```python
+from exports.db_factory import DBFactory
+from pathlib import Path
+
+dotenv_path = (Path(__file__).parent / '.env').resolve()
+factory = DBFactory(db_name='hub_db', app_env='PROD', dotenv_path=dotenv_path, city=None)
+```
+
+
+## Create Database Tables ##
+Use the *DBSetUp* class in the persistence package to create the required database tables as described below
+```python
+from persistence import DBSetup
+from pathlib import Path
+
+dotenv_path = (Path(__file__).parent / '.env').resolve()
+DBSetup(db_name='hub_db', app_env='PROD', dotenv_path=dotenv_path)
+```

exports/db_factory.py

@@ -13,10 +13,9 @@ class DBFactory:
   DBFactory class
   """
 
-  def __init__(self, city, db_name, app_env):
+  def __init__(self, db_name, app_env, dotenv_path):
-    self._city = city
+    self._city_repo = CityRepo(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
-    self._city_repo = CityRepo(db_name=db_name, app_env=app_env)
+    self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
-    self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, app_env=app_env)
 
   def get_city(self, city_id):
     """

exports/energy_systems/heat_pump_export.py

@@ -56,7 +56,7 @@ class HeatPumpExport:
       insel_file_handler.write(insel_template)
       # Now run insel
       self._delete_existing_output_files()
-      os.system('insel {}'.format(insel_file))
+      os.system('/usr/local/bin/insel {}'.format(insel_file))
       # Writer headers to csv output files generated by insel
       self._write_insel_output_headers()
       # User output

imports/db_factory.py

@@ -14,10 +14,10 @@ class DBFactory:
   DBFactory class
   """
 
-  def __init__(self, city, db_name, app_env):
+  def __init__(self, city, db_name, dotenv_path, app_env):
     self._city = city
-    self._city_repo = CityRepo(db_name=db_name, app_env=app_env)
+    self._city_repo = CityRepo(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env)
-    self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, app_env=app_env)
+    self._hp_simulation_repo = HeatPumpSimulationRepo(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env)
 
   def persist_city(self):
     """

install_postgresql_linux.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+sudo apt-get update
+sudo apt-get install postgresql

persistence/__init__.py

@@ -2,3 +2,4 @@ from .base_repo import BaseRepo
 from .repositories.city_repo import CityRepo
 from .repositories.building_repo import BuildingRepo
 from .repositories.heat_pump_simulation_repo import HeatPumpSimulationRepo
+from .db_setup import DBSetup

persistence/base_repo.py

@@ -12,17 +12,14 @@ from sqlalchemy.orm import Session
 
 class BaseRepo:
 
-  def __init__(self, db_name, app_env='TEST'):
+  def __init__(self, db_name, dotenv_path: str, app_env='TEST'):
-    self.config = BaseConfiguration(db_name, app_env)
+    try:
+      self.config = BaseConfiguration(db_name, dotenv_path, app_env)
       self.engine = create_engine(self.config.conn_string())
       self.session = Session(self.engine)
-
+    except ValueError as err:
-  def __del__(self):
+      print(f'Missing value for credentials: {err}')
-    """
+
-    Close database sessions
-    :return:
-    """
-    self.session.close()
 
 
 

persistence/db_config.py

@@ -9,28 +9,31 @@ import os
 from dotenv import load_dotenv
 from sqlalchemy.ext.declarative import declarative_base
 
-
 Base = declarative_base()
 
-# load environmental variables
-load_dotenv()
-
 
 class BaseConfiguration(object):
   """
     Base configuration class to hold common persistence configuration
     """
-    def __init__(self, db_name: str, app_env='TEST'):
+
+  def __init__(self, db_name: str, dotenv_path: str, app_env='TEST'):
     """
       :param db_name: database name
       :param app_env: application environment, test or production
+      :param dotenv_path: the absolute path to dotenv file
       """
+    try:
+      # load environmental variables
+      load_dotenv(dotenv_path=dotenv_path)
       self._db_name = db_name
       self._db_host = os.getenv(f'{app_env}_DB_HOST')
       self._db_user = os.getenv(f'{app_env}_DB_USER')
       self._db_pass = os.getenv(f'{app_env}_DB_PASSWORD')
       self._db_port = os.getenv(f'{app_env}_DB_PORT')
       self.hub_token = os.getenv('HUB_TOKEN')
+    except KeyError as err:
+      print(f'Error with credentials: {err}')
 
   def conn_string(self):
     """

persistence/db_setup.py

@@ -0,0 +1,11 @@
+from persistence.models import City
+from persistence import BaseRepo
+from persistence.models import HeatPumpSimulation
+
+
+class DBSetup:
+
+  def __init__(self, db_name, app_env, dotenv_path):
+    repo = BaseRepo(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path)
+    City.__table__.create(bind=repo.engine, checkfirst=True)
+    HeatPumpSimulation.__table__.create(bind=repo.engine, checkfirst=True)

persistence/repositories/city_repo.py

@@ -19,10 +19,10 @@ from typing import Union, Dict
 class CityRepo(BaseRepo):
   _instance = None
 
-  def __init__(self, db_name, app_env):
+  def __init__(self, db_name: str, dotenv_path: str, app_env: str):
-    super().__init__(db_name, app_env)
+    super().__init__(db_name,  dotenv_path, app_env)
 
-  def __new__(cls, db_name, app_env):
+  def __new__(cls, db_name, dotenv_path, app_env):
     """
     Implemented for a singleton pattern
     """

persistence/repositories/heat_pump_simulation_repo.py

@@ -15,11 +15,11 @@ from typing import Union, Dict
 class HeatPumpSimulationRepo(BaseRepo):
   _instance = None
 
-  def __init__(self, db_name, app_env):
+  def __init__(self, db_name, dotenv_path, app_env):
-    super().__init__(db_name, app_env)
+    super().__init__(db_name, dotenv_path, app_env)
-    self._city_repo = CityRepo(db_name, app_env)
+    self._city_repo = CityRepo(db_name, dotenv_path, app_env)
 
-  def __new__(cls, db_name, app_env):
+  def __new__(cls, db_name, dotenv_path, app_env):
     """
     Implemented for a singleton pattern
     """

unittests/test_db_factory.py

@@ -27,7 +27,7 @@ class TestDBFactory(TestCase):
     :return: None
     """
     # Create test database
-    repo = BaseRepo(db_name='test_db', app_env='TEST')
+    repo = BaseRepo(db_name='test_db', app_env='TEST', dotenv_path='../.env')
     eng = create_engine(f'postgresql://{repo.config.get_db_user()}@/{repo.config.get_db_user()}')
 
     try:
@@ -48,8 +48,8 @@ class TestDBFactory(TestCase):
 
     city_file = "../unittests/tests_data/C40_Final.gml"
     cls.city = GeometryFactory('citygml', city_file).city
-    cls._db_factory = DBFactory(city=cls.city, db_name='test_db', app_env='TEST')
+    cls._db_factory = DBFactory(city=cls.city, db_name='test_db', app_env='TEST', dotenv_path='../.env')
-    cls._export_db_factory = ExportDBFactory(city=cls.city, db_name='test_db', app_env='TEST')
+    cls._export_db_factory = ExportDBFactory(db_name='test_db', app_env='TEST', dotenv_path='../.env')
 
   def test_save_city(self):
     saved_city = self._db_factory.persist_city()

unittests/test_heat_pump_simulation.py

@@ -44,7 +44,7 @@ class TestHeatPumpSimulation(TestCase):
     Test setup
     :return: None
     """
-    repo = BaseRepo(db_name='test_db', app_env='TEST')
+    repo = BaseRepo(db_name='test_db', app_env='TEST', dotenv_path='../.env')
     eng = create_engine(f'postgresql://{repo.config.get_db_user()}@/{repo.config.get_db_user()}')
 
     try:
@@ -68,8 +68,8 @@ class TestHeatPumpSimulation(TestCase):
     cls._city = GeometryFactory('citygml', city_file).city
     EnergySystemsFactory('air source hp', cls._city).enrich()
 
-    cls._db_factory = DBFactory(city=cls._city, db_name='test_db', app_env='TEST')
+    cls._db_factory = DBFactory(city=cls._city, db_name='test_db', app_env='TEST', dotenv_path='../.env')
-    cls._export_db_factory = ExportDBFactory(city=cls._city, db_name='test_db', app_env='TEST')
+    cls._export_db_factory = ExportDBFactory(db_name='test_db', app_env='TEST', dotenv_path='../.env')
 
   def test_heat_pump_simulation_persistence(self):
     output = EnergySystemsExportFactory(city=self._city, user_input=hp_sim_data, hp_model='018',