From a296769361ea19326abab41af12d8cfd6fcf418f Mon Sep 17 00:00:00 2001 From: r_sanchez Date: Tue, 24 Oct 2023 11:47:35 -0400 Subject: [PATCH] Contents from the hub persistence API --- README.md | 52 ++++++ __init__.py | 0 configuration.py | 67 ++++++++ db_control.py | 249 +++++++++++++++++++++++++++++ db_setup.py | 70 ++++++++ models/__init__.py | 8 + models/application.py | 32 ++++ models/city.py | 36 +++++ models/city_object.py | 80 +++++++++ models/simulation_results.py | 33 ++++ models/user.py | 42 +++++ repositories/__init__.py | 3 + repositories/application.py | 111 +++++++++++++ repositories/city.py | 139 ++++++++++++++++ repositories/city_object.py | 133 +++++++++++++++ repositories/simulation_results.py | 169 ++++++++++++++++++++ repositories/user.py | 159 ++++++++++++++++++ repository.py | 22 +++ 18 files changed, 1405 insertions(+) create mode 100644 README.md create mode 100644 __init__.py create mode 100644 configuration.py create mode 100644 db_control.py create mode 100644 db_setup.py create mode 100644 models/__init__.py create mode 100644 models/application.py create mode 100644 models/city.py create mode 100644 models/city_object.py create mode 100644 models/simulation_results.py create mode 100644 models/user.py create mode 100644 repositories/__init__.py create mode 100644 repositories/application.py create mode 100644 repositories/city.py create mode 100644 repositories/city_object.py create mode 100644 repositories/simulation_results.py create mode 100644 repositories/user.py create mode 100644 repository.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..bff34cf --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +## Database Persistence ## +The persistence package includes classes to store different class objects in a Postgres database. + +### models ### +This defines models for all class objects that we want to persist. It is used for Object Relation Mapping (ORM) +of the class objects to database table columns + +### repositories ### +This defines repository classes that contain CRUD methods for database operations. The constructor of all repositories requires +The database name to connect to and the application environment (PROD or TEST). Tests use a different database +from the production environment, which is why this is necessary. An example is shown below + +```python +from hub.persistence import CityRepo + +# instantiate city repo for hub production database +city_repo = CityRepo(db_name='hub', app_env='PROD') +``` +All database operations are conducted with the production database (*PROD*) named *hub* in the example above + +### config_db ## +This Python file is a configuration class that contains variables that map to configuration parameters in a .env file. +It also contains a method ``def conn_string()`` which returns the connection string to a Postgres database. + +### Base ## +This class has a constructor that establishes a database connection and returns a reference for database-related CRUD operations. + +### Database Configuration Parameter ### +A .env file (or environment variables) with configuration parameters described below are needed to establish a database connection: +``` +# production database credentials +PROD_DB_USER=postgres-database-user +PROD_DB_PASSWORD=postgres-database-password +PROD_DB_HOST=database-host +PROD_DB_PORT=database-port + +# test database credentials +TEST_DB_USER=postgres-database-user +TEST_DB_PASSWORD=postgres-database-password +TEST_DB_HOST=database-host +TEST_DB_PORT=database-port +``` + +### Database Related Unit Test +Unit tests that involve database operations require a Postgres database to be set up. +The tests connect to the database server using the default postgres user (*postgres*). +NB: You can provide any credentials for the test to connect to postgres, just make sure +the credentials are set in your .env file as explained above in *Database Configuration Parameters* section + +When the tests are run, a **test_db** database is created and then the required tables for +the test. Before the tests run, the *test_db* is deleted to ensure that each test starts +on a clean slate diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/configuration.py b/configuration.py new file mode 100644 index 0000000..a71ca59 --- /dev/null +++ b/configuration.py @@ -0,0 +1,67 @@ +""" +Persistence (Postgresql) configuration +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Peter Yefi peteryefi@gmail.com +""" + +import logging +import os +from pathlib import Path +from dotenv import load_dotenv +from sqlalchemy.ext.declarative import declarative_base + +Models = declarative_base() + + +class Configuration: + """ + Configuration class to hold common persistence configuration + """ + + def __init__(self, db_name: str, dotenv_path: str, app_env='TEST'): + """ + :param db_name: database name + :param app_env: application environment, test or production + :param dotenv_path: the absolute path to dotenv file + """ + try: + # load environmental variables + if not Path(dotenv_path).exists(): + error_message = f'dotenv file doesn\'t exists at {dotenv_path}' + logging.error(error_message) + raise FileNotFoundError(error_message) + load_dotenv(dotenv_path=dotenv_path) + + self._db_name = db_name + self._db_host = os.getenv(f'{app_env}_DB_HOST') + self._db_user = os.getenv(f'{app_env}_DB_USER') + self._db_pass = os.getenv(f'{app_env}_DB_PASSWORD') + self._db_port = os.getenv(f'{app_env}_DB_PORT') + self.hub_token = os.getenv('HUB_TOKEN') + except KeyError as err: + logging.error('Error with credentials: %s', err) + + @property + def connection_string(self): + """ + Returns a connection string postgresql + :return: connection string + """ + if self._db_pass: + return f'postgresql://{self._db_user}:{self._db_pass}@{self._db_host}:{self._db_port}/{self._db_name}' + return f'postgresql://{self._db_user}@{self._db_host}:{self._db_port}/{self._db_name}' + + @property + def db_user(self): + """ + retrieve the configured username + """ + return self._db_user + + @property + def db_name(self): + """ + retrieve the configured database name + """ + return self._db_name diff --git a/db_control.py b/db_control.py new file mode 100644 index 0000000..f53d0d3 --- /dev/null +++ b/db_control.py @@ -0,0 +1,249 @@ +""" +DBFactory performs read related operations +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project CoderPeter Yefi peteryefi@gmail.com +""" +import json +from typing import Dict + + +from hub.persistence.repositories.application import Application +from hub.persistence.repositories.city import City +from hub.persistence.repositories.city_object import CityObject +from hub.persistence.repositories.simulation_results import SimulationResults +from hub.persistence.repositories.user import User +from hub.persistence.repositories.user import UserRoles + + +class DBControl: + """ + DBFactory class + """ + + def __init__(self, db_name, app_env, dotenv_path): + self._city = City(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env) + self._application = Application(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + self._user = User(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + self._city_object = CityObject(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + self._simulation_results = SimulationResults(db_name=db_name, dotenv_path=dotenv_path, app_env=app_env) + + def application_info(self, application_uuid) -> Application: + """ + Retrieve the application info for the given uuid from the database + :param application_uuid: the uuid for the application + :return: Application + """ + return self._application.get_by_uuid(application_uuid) + + def user_info(self, name, password, application_id) -> User: + """ + Retrieve the user info for the given name and password and application_id from the database + :param name: the username + :param password: the user password + :param application_id: the application id + :return: User + """ + return self._user.get_by_name_application_id_and_password(name, password, application_id) + + def user_login(self, name, password, application_uuid) -> User: + """ + Retrieve the user info from the database + :param name: the username + :param password: the user password + :param application_uuid: the application uuid + :return: User + """ + return self._user.get_by_name_application_uuid_and_password(name, password, application_uuid) + + def cities_by_user_and_application(self, user_id, application_id) -> [City]: + """ + Retrieve the cities belonging to the user and the application from the database + :param user_id: User id + :param application_id: Application id + :return: [City] + """ + return self._city.get_by_user_id_and_application_id(user_id, application_id) + + def building(self, name, user_id, application_id, scenario) -> CityObject: + """ + Retrieve the building from the database + :param name: Building name + :param user_id: User id + :param application_id: Application id + :param scenario: Scenario + : + """ + cities = self._city.get_by_user_id_application_id_and_scenario(user_id, application_id, scenario) + for city in cities: + result = self.building_info(name, city[0].id) + if result is not None: + return result + return None + + def building_info(self, name, city_id) -> CityObject: + """ + Retrieve the building info from the database + :param name: Building name + :param city_id: City ID + :return: CityObject + """ + return self._city_object.get_by_name_or_alias_and_city(name, city_id) + + def buildings_info(self, request_values, city_id) -> [CityObject]: + """ + Retrieve the buildings info from the database + :param request_values: Building names + :param city_id: City ID + :return: [CityObject] + """ + buildings = [] + for name in request_values['names']: + buildings.append(self.building_info(name, city_id)) + return buildings + + def results(self, user_id, application_id, request_values, result_names=None) -> Dict: + """ + Retrieve the simulation results for the given cities from the database + :param user_id: the user id owning the results + :param application_id: the application id owning the results + :param request_values: dictionary containing the scenario and building names to grab the results + :param result_names: if given, filter the results to the selected names + """ + if result_names is None: + result_names = [] + results = {} + for scenario in request_values['scenarios']: + for scenario_name in scenario.keys(): + result_sets = self._city.get_by_user_id_application_id_and_scenario( + user_id, + application_id, + scenario_name + ) + if result_sets is None: + continue + for result_set in result_sets: + city_id = result_set[0].id + + results[scenario_name] = [] + for building_name in scenario[scenario_name]: + _building = self._city_object.get_by_name_or_alias_and_city(building_name, city_id) + if _building is None: + continue + city_object_id = _building.id + _ = self._simulation_results.get_simulation_results_by_city_id_city_object_id_and_names( + city_id, + city_object_id, + result_names) + + for value in _: + values = json.loads(value.values) + values["building"] = building_name + results[scenario_name].append(values) + return results + + def persist_city(self, city: City, pickle_path, scenario, application_id: int, user_id: int): + """ + Creates a city into the database + :param city: City to be stored + :param pickle_path: Path to save the pickle file + :param scenario: Simulation scenario name + :param application_id: Application id owning this city + :param user_id: User who create the city + return identity_id + """ + return self._city.insert(city, pickle_path, scenario, application_id, user_id) + + def update_city(self, city_id, city): + """ + Update an existing city in the database + :param city_id: the id of the city to update + :param city: the updated city object + """ + return self._city.update(city_id, city) + + def persist_application(self, name: str, description: str, application_uuid: str): + """ + Creates information for an application in the database + :param name: name of application + :param description: the description of the application + :param application_uuid: the uuid of the application to be created + """ + return self._application.insert(name, description, application_uuid) + + def update_application(self, name: str, description: str, application_uuid: str): + """ + Update the application information stored in the database + :param name: name of application + :param description: the description of the application + :param application_uuid: the uuid of the application to be created + """ + return self._application.update(application_uuid, name, description) + + def add_simulation_results(self, name, values, city_id=None, city_object_id=None): + """ + Add simulation results to the city or to the city_object to the database + :param name: simulation and simulation engine name + :param values: simulation values in json format + :param city_id: city id or None + :param city_object_id: city object id or None + """ + return self._simulation_results.insert(name, values, city_id, city_object_id) + + def create_user(self, name: str, application_id: int, password: str, role: UserRoles): + """ + Creates a new user in the database + :param name: the name of the user + :param application_id: the application id of the user + :param password: the password of the user + :param role: the role of the user + """ + return self._user.insert(name, password, role, application_id) + + def update_user(self, user_id: int, name: str, password: str, role: UserRoles): + """ + Updates a user in the database + :param user_id: the id of the user + :param name: the name of the user + :param password: the password of the user + :param role: the role of the user + """ + return self._user.update(user_id, name, password, role) + + def get_by_name_and_application(self, name: str, application: int): + """ + Retrieve a single user from the database + :param name: username + :param application: application accessing hub + """ + return self._user.get_by_name_and_application(name, application) + + def delete_user(self, user_id): + """ + Delete a single user from the database + :param user_id: the id of the user to delete + """ + self._user.delete(user_id) + + def delete_city(self, city_id): + """ + Deletes a single city from the database + :param city_id: the id of the city to get + """ + self._city.delete(city_id) + + def delete_results_by_name(self, name, city_id=None, city_object_id=None): + """ + Deletes city object simulation results from the database + :param name: simulation name + :param city_id: if given, delete delete the results for the city with id city_id + :param city_object_id: if given, delete delete the results for the city object with id city_object_id + """ + self._simulation_results.delete(name, city_id=city_id, city_object_id=city_object_id) + + def delete_application(self, application_uuid): + """ + Deletes a single application from the database + :param application_uuid: the id of the application to get + """ + self._application.delete(application_uuid) diff --git a/db_setup.py b/db_setup.py new file mode 100644 index 0000000..cb8bf33 --- /dev/null +++ b/db_setup.py @@ -0,0 +1,70 @@ +""" +Database setup +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Peter Yefi peteryefi@gmail.com +""" + +import logging +from hub.persistence.repository import Repository +from hub.persistence.models import Application +from hub.persistence.models import City +from hub.persistence.models import CityObject +from hub.persistence.models import User +from hub.persistence.models import UserRoles +from hub.persistence.models import SimulationResults +from hub.persistence.repositories.user import User as UserRepository +from hub.persistence.repositories.application import Application as ApplicationRepository + + +class DBSetup: + """ + Creates a Persistence database structure + """ + + def __init__(self, db_name, app_env, dotenv_path, admin_password, application_uuid): + """ + Creates database tables a default admin user and a default admin app with the given password and uuid + :param db_name: database name + :param app_env: application environment type [TEST|PROD] + :param dotenv_path: .env file path + :param admin_password: administrator password for the application uuid + :application_uuid: application uuid + """ + repository = Repository(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + + # Create the tables using the models + Application.__table__.create(bind=repository.engine, checkfirst=True) + User.__table__.create(bind=repository.engine, checkfirst=True) + City.__table__.create(bind=repository.engine, checkfirst=True) + CityObject.__table__.create(bind=repository.engine, checkfirst=True) + SimulationResults.__table__.create(bind=repository.engine, checkfirst=True) + + self._user_repo = UserRepository(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + self._application_repo = ApplicationRepository(db_name=db_name, app_env=app_env, dotenv_path=dotenv_path) + application_id = self._create_admin_app(self._application_repo, application_uuid) + self._create_admin_user(self._user_repo, admin_password, application_id) + + @staticmethod + def _create_admin_app(application_repo, application_uuid): + name = 'AdminTool' + description = 'Admin tool to control city persistence and to test the API v1.4' + logging.info('Creating default admin tool application...') + application = application_repo.insert(name, description, application_uuid) + + if isinstance(application, dict): + logging.info(application) + else: + msg = f'Created Admin tool with application_uuid: {application_uuid}' + logging.info(msg) + return application.id + + @staticmethod + def _create_admin_user(user_repo, admin_password, application_id): + password = admin_password + logging.info('Creating default admin user...') + user = user_repo.insert('Administrator', password, UserRoles.Admin, application_id) + if isinstance(user, dict): + logging.info(user) + else: + logging.info('Created Admin user') diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..4682044 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1,8 @@ +""" +Models package +""" +from .application import Application +from .city import City +from .city_object import CityObject +from .simulation_results import SimulationResults +from .user import User, UserRoles diff --git a/models/application.py b/models/application.py new file mode 100644 index 0000000..23f05a2 --- /dev/null +++ b/models/application.py @@ -0,0 +1,32 @@ +""" +Model representation of an application +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca +""" + +import datetime + +from sqlalchemy import Column, Integer, String, Sequence +from sqlalchemy import DateTime +from sqlalchemy.dialects.postgresql import UUID + +from hub.persistence.configuration import Models + + +class Application(Models): + """ + A model representation of an application + """ + __tablename__ = 'application' + id = Column(Integer, Sequence('application_id_seq'), primary_key=True) + name = Column(String, nullable=False) + description = Column(String, nullable=False) + application_uuid = Column(UUID(as_uuid=True), nullable=False) + created = Column(DateTime, default=datetime.datetime.utcnow) + updated = Column(DateTime, default=datetime.datetime.utcnow) + + def __init__(self, name, description, application_uuid): + self.name = name + self.description = description + self.application_uuid = application_uuid diff --git a/models/city.py b/models/city.py new file mode 100644 index 0000000..35f529b --- /dev/null +++ b/models/city.py @@ -0,0 +1,36 @@ +""" +Model representation of a City +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Peter Yefi peteryefi@gmail.com +""" + +import datetime + +from sqlalchemy import Column, Integer, String, Sequence, ForeignKey +from sqlalchemy import DateTime + +from hub.persistence.configuration import Models + + +class City(Models): + """A model representation of a city + """ + __tablename__ = 'city' + id = Column(Integer, Sequence('city_id_seq'), primary_key=True) + pickle_path = Column(String, nullable=False) + name = Column(String, nullable=False) + scenario = Column(String, nullable=False) + application_id = Column(Integer, ForeignKey('application.id', ondelete='CASCADE'), nullable=False) + user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE'), nullable=True) + hub_release = Column(String, nullable=False) + created = Column(DateTime, default=datetime.datetime.utcnow) + updated = Column(DateTime, default=datetime.datetime.utcnow) + + def __init__(self, pickle_path, name, scenario, application_id, user_id, hub_release): + self.pickle_path = str(pickle_path) + self.name = name + self.scenario = scenario + self.application_id = application_id + self.user_id = user_id + self.hub_release = hub_release diff --git a/models/city_object.py b/models/city_object.py new file mode 100644 index 0000000..1d88b9e --- /dev/null +++ b/models/city_object.py @@ -0,0 +1,80 @@ +""" +Model representation of a city object +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca +""" + +import datetime +import logging + +from sqlalchemy import Column, Integer, String, Sequence, ForeignKey, Float +from sqlalchemy import DateTime + +from hub.city_model_structure.building import Building +from hub.persistence.configuration import Models + + +class CityObject(Models): + """ + A model representation of an application + """ + __tablename__ = 'city_object' + id = Column(Integer, Sequence('city_object_id_seq'), primary_key=True) + city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=False) + name = Column(String, nullable=False) + aliases = Column(String, nullable=True) + type = Column(String, nullable=False) + year_of_construction = Column(Integer, nullable=True) + function = Column(String, nullable=True) + usage = Column(String, nullable=True) + volume = Column(Float, nullable=False) + area = Column(Float, nullable=False) + total_heating_area = Column(Float, nullable=False) + wall_area = Column(Float, nullable=False) + windows_area = Column(Float, nullable=False) + roof_area = Column(Float, nullable=False) + total_pv_area = Column(Float, nullable=False) + system_name = Column(String, nullable=False) + created = Column(DateTime, default=datetime.datetime.utcnow) + updated = Column(DateTime, default=datetime.datetime.utcnow) + + def __init__(self, city_id, building: Building): + self.city_id = city_id + self.name = building.name + self.aliases = building.aliases + self.type = building.type + self.year_of_construction = building.year_of_construction + self.function = building.function + self.usage = building.usages_percentage + self.volume = building.volume + self.area = building.floor_area + self.roof_area = sum(roof.solid_polygon.area for roof in building.roofs) + self.total_pv_area = sum(roof.solid_polygon.area * roof.solar_collectors_area_reduction_factor for roof in building.roofs) + storeys = building.storeys_above_ground + wall_area = 0 + window_ratio = 0 + try: + if storeys is None: + storeys = building.max_height / building.average_storey_height + for internal_zone in building.internal_zones: + for thermal_zone in internal_zone.thermal_zones_from_internal_zones: + for thermal_boundary in thermal_zone.thermal_boundaries: + window_ratio = thermal_boundary.window_ratio + break + except TypeError: + storeys = 0 + logging.warning( + 'building %s has no storey height so heating area, storeys and window ratio cannot be calculated', + self.name + ) + self.total_heating_area = building.floor_area * storeys + + for wall in building.walls: + wall_area += wall.solid_polygon.area + self.wall_area = wall_area + self.windows_area = wall_area * window_ratio + system_name = building.energy_systems_archetype_name + if system_name is None: + system_name = '' + self.system_name = system_name diff --git a/models/simulation_results.py b/models/simulation_results.py new file mode 100644 index 0000000..a40c077 --- /dev/null +++ b/models/simulation_results.py @@ -0,0 +1,33 @@ +""" +Model representation of simulation results +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca +""" + +import datetime + +from sqlalchemy import Column, Integer, String, Sequence, ForeignKey +from sqlalchemy import DateTime +from sqlalchemy.dialects.postgresql import JSONB +from hub.persistence.configuration import Models + + +class SimulationResults(Models): + """ + A model representation of an application + """ + __tablename__ = 'simulation_results' + id = Column(Integer, Sequence('simulation_results_id_seq'), primary_key=True) + city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=True) + city_object_id = Column(Integer, ForeignKey('city_object.id', ondelete='CASCADE'), nullable=True) + name = Column(String, nullable=False) + values = Column(JSONB, nullable=False) + created = Column(DateTime, default=datetime.datetime.utcnow) + updated = Column(DateTime, default=datetime.datetime.utcnow) + + def __init__(self, name, values, city_id=None, city_object_id=None): + self.name = name + self.values = values + self.city_id = city_id + self.city_object_id = city_object_id diff --git a/models/user.py b/models/user.py new file mode 100644 index 0000000..7a45e0b --- /dev/null +++ b/models/user.py @@ -0,0 +1,42 @@ +""" +Model representation of a User +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Peter Yefi peteryefi@gmail.com +""" + +import datetime +import enum + +from sqlalchemy import Column, Integer, String, Sequence +from sqlalchemy import DateTime, Enum + +from hub.persistence.configuration import Models + + +class UserRoles(enum.Enum): + """ + User roles enum + """ + Admin = 'Admin' + Hub_Reader = 'Hub_Reader' + + +class User(Models): + """ + A model representation of a city + """ + __tablename__ = 'user' + id = Column(Integer, Sequence('user_id_seq'), primary_key=True) + name = Column(String, nullable=False) + password = Column(String, nullable=False) + role = Column(Enum(UserRoles), nullable=False, default=UserRoles.Hub_Reader) + application_id = Column(Integer, nullable=False) + created = Column(DateTime, default=datetime.datetime.utcnow) + updated = Column(DateTime, default=datetime.datetime.utcnow) + + def __init__(self, name, password, role, application_id): + self.name = name + self.password = password + self.role = role + self.application_id = application_id diff --git a/repositories/__init__.py b/repositories/__init__.py new file mode 100644 index 0000000..2fb3351 --- /dev/null +++ b/repositories/__init__.py @@ -0,0 +1,3 @@ +""" +Repositories Package +""" diff --git a/repositories/application.py b/repositories/application.py new file mode 100644 index 0000000..a70fde3 --- /dev/null +++ b/repositories/application.py @@ -0,0 +1,111 @@ +""" +Application repository with database CRUD operations +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca +""" + +import datetime +import logging + +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm.session import Session + +from hub.persistence.repository import Repository +from hub.persistence.models import Application as Model + + +class Application(Repository): + """ + Application repository + """ + _instance = None + + def __init__(self, db_name: str, dotenv_path: str, app_env: str): + super().__init__(db_name, dotenv_path, app_env) + + def __new__(cls, db_name, dotenv_path, app_env): + """ + Implemented for a singleton pattern + """ + if cls._instance is None: + cls._instance = super(Application, cls).__new__(cls) + return cls._instance + + def insert(self, name: str, description: str, application_uuid: str): + """ + Inserts a new application + :param name: Application name + :param description: Application description + :param application_uuid: Unique identifier for the application + :return: Identity id + """ + try: + application = self.get_by_uuid(application_uuid) + if application is not None: + raise SQLAlchemyError('application already exists') + except TypeError: + pass + try: + application = Model(name=name, description=description, application_uuid=application_uuid) + with Session(self.engine) as session: + session.add(application) + session.commit() + session.refresh(application) + return application.id + except SQLAlchemyError as err: + logging.error('An error occurred while creating application %s', err) + raise SQLAlchemyError from err + + def update(self, application_uuid: str, name: str, description: str): + """ + Updates an application + :param application_uuid: the application uuid of the application to be updated + :param name: the application name + :param description: the application description + :return: None + """ + try: + with Session(self.engine) as session: + session.query(Model).filter( + Model.application_uuid == application_uuid + ).update({'name': name, 'description': description, 'updated': datetime.datetime.utcnow()}) + session.commit() + except SQLAlchemyError as err: + logging.error('Error while updating application %s', err) + raise SQLAlchemyError from err + + def delete(self, application_uuid: str): + """ + Deletes an application with the application_uuid + :param application_uuid: The application uuid + :return: None + """ + try: + with Session(self.engine) as session: + session.query(Model).filter(Model.application_uuid == application_uuid).delete() + session.flush() + session.commit() + except SQLAlchemyError as err: + logging.error('Error while deleting application %s', err) + raise SQLAlchemyError from err + + def get_by_uuid(self, application_uuid: str) -> Model: + """ + Fetch Application based on the application uuid + :param application_uuid: the application uuid + :return: Application with the provided application_uuid + """ + try: + with Session(self.engine) as session: + result_set = session.execute(select(Model).where( + Model.application_uuid == application_uuid) + ).first() + return result_set[0] + except SQLAlchemyError as err: + logging.error('Error while fetching application by application_uuid %s', err) + raise SQLAlchemyError from err + except TypeError as err: + logging.error('Error while fetching application, empty result %s', err) + raise TypeError from err diff --git a/repositories/city.py b/repositories/city.py new file mode 100644 index 0000000..d1e0734 --- /dev/null +++ b/repositories/city.py @@ -0,0 +1,139 @@ +""" +City repository with database CRUD operations +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Peter Yefi peteryefi@gmail.com +""" +import datetime +import logging + +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from hub.city_model_structure.city import City as CityHub +from hub.persistence.repository import Repository +from hub.persistence.models import City as Model +from hub.persistence.models import CityObject +from hub.version import __version__ + + +class City(Repository): + """ + City repository + """ + _instance = None + + def __init__(self, db_name: str, dotenv_path: str, app_env: str): + super().__init__(db_name, dotenv_path, app_env) + + def __new__(cls, db_name, dotenv_path, app_env): + """ + Implemented for a singleton pattern + """ + if cls._instance is None: + cls._instance = super(City, cls).__new__(cls) + return cls._instance + + def insert(self, city: CityHub, pickle_path, scenario, application_id, user_id: int): + """ + Inserts a city + :param city: The complete city instance + :param pickle_path: Path to the pickle + :param scenario: Simulation scenario name + :param application_id: Application id owning the instance + :param user_id: User id owning the instance + :return: Identity id + """ + city.save_compressed(pickle_path) + try: + db_city = Model( + pickle_path, + city.name, + scenario, + application_id, + user_id, + __version__) + with Session(self.engine) as session: + session.add(db_city) + session.flush() + session.commit() + for building in city.buildings: + db_city_object = CityObject(db_city.id, + building) + session.add(db_city_object) + session.flush() + session.commit() + session.refresh(db_city) + return db_city.id + except SQLAlchemyError as err: + logging.error('An error occurred while creating a city %s', err) + raise SQLAlchemyError from err + + def update(self, city_id: int, city: CityHub): + """ + Updates a city name (other updates makes no sense) + :param city_id: the id of the city to be updated + :param city: the city object + :return: None + """ + try: + now = datetime.datetime.utcnow() + with Session(self.engine) as session: + session.query(Model).filter(Model.id == city_id).update({'name': city.name, 'updated': now}) + session.commit() + except SQLAlchemyError as err: + logging.error('Error while updating city %s', err) + raise SQLAlchemyError from err + + def delete(self, city_id: int): + """ + Deletes a City with the id + :param city_id: the city id + :return: None + """ + try: + with Session(self.engine) as session: + session.query(CityObject).filter(CityObject.city_id == city_id).delete() + session.query(Model).filter(Model.id == city_id).delete() + session.commit() + except SQLAlchemyError as err: + logging.error('Error while fetching city %s', err) + raise SQLAlchemyError from err + + def get_by_user_id_application_id_and_scenario(self, user_id, application_id, scenario) -> [Model]: + """ + Fetch city based on the user who created it + :param user_id: the user id + :param application_id: the application id + :param scenario: simulation scenario name + :return: [ModelCity] + """ + try: + with Session(self.engine) as session: + result_set = session.execute(select(Model).where(Model.user_id == user_id, + Model.application_id == application_id, + Model.scenario == scenario + )).all() + return result_set + except SQLAlchemyError as err: + logging.error('Error while fetching city by name %s', err) + raise SQLAlchemyError from err + + def get_by_user_id_and_application_id(self, user_id, application_id) -> [Model]: + """ + Fetch city based on the user who created it + :param user_id: the user id + :param application_id: the application id + :return: ModelCity + """ + try: + with Session(self.engine) as session: + result_set = session.execute( + select(Model).where(Model.user_id == user_id, Model.application_id == application_id) + ) + return [r[0] for r in result_set] + except SQLAlchemyError as err: + logging.error('Error while fetching city by name %s', err) + raise SQLAlchemyError from err + diff --git a/repositories/city_object.py b/repositories/city_object.py new file mode 100644 index 0000000..3028741 --- /dev/null +++ b/repositories/city_object.py @@ -0,0 +1,133 @@ +""" +City Object repository with database CRUD operations +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca +""" +import datetime +import logging + +from sqlalchemy import select, or_ +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from hub.city_model_structure.building import Building +from hub.persistence.repository import Repository +from hub.persistence.models import CityObject as Model + + +class CityObject(Repository): + """ + City object repository + """ + _instance = None + + def __init__(self, db_name: str, dotenv_path: str, app_env: str): + super().__init__(db_name, dotenv_path, app_env) + + def __new__(cls, db_name, dotenv_path, app_env): + """ + Implemented for a singleton pattern + """ + if cls._instance is None: + cls._instance = super(CityObject, cls).__new__(cls) + return cls._instance + + def insert(self, city_id: int, building: Building): + """ + Inserts a new city object + :param city_id: city id for the city owning this city object + :param building: the city object (only building for now) to be inserted + return Identity id + """ + city_object = self.get_by_name_or_alias_and_city(building.name, city_id) + if city_object is not None: + raise SQLAlchemyError(f'A city_object named {building.name} already exists in that city') + try: + city_object = Model(city_id=city_id, + building=building) + with Session(self.engine) as session: + session.add(city_object) + session.flush() + session.commit() + session.refresh(city_object) + return city_object.id + except SQLAlchemyError as err: + logging.error('An error occurred while creating city_object %s', err) + raise SQLAlchemyError from err + + def update(self, city_id: int, building: Building): + """ + Updates an application + :param city_id: the city id of the city owning the city object + :param building: the city object + :return: None + """ + try: + object_usage = '' + for internal_zone in building.internal_zones: + for usage in internal_zone.usages: + object_usage = f'{object_usage}{usage.name}_{usage.percentage} ' + object_usage = object_usage.rstrip() + with Session(self.engine) as session: + session.query(Model).filter(Model.name == building.name, Model.city_id == city_id).update( + {'name': building.name, + 'alias': building.alias, + 'object_type': building.type, + 'year_of_construction': building.year_of_construction, + 'function': building.function, + 'usage': object_usage, + 'volume': building.volume, + 'area': building.floor_area, + 'updated': datetime.datetime.utcnow()}) + session.commit() + except SQLAlchemyError as err: + logging.error('Error while updating city object %s', err) + raise SQLAlchemyError from err + + def delete(self, city_id: int, name: str): + """ + Deletes an application with the application_uuid + :param city_id: The id for the city owning the city object + :param name: The city object name + :return: None + """ + try: + with Session(self.engine) as session: + session.query(Model).filter(Model.city_id == city_id, Model.name == name).delete() + session.commit() + except SQLAlchemyError as err: + logging.error('Error while deleting application %s', err) + raise SQLAlchemyError from err + + def get_by_name_or_alias_and_city(self, name, city_id) -> Model: + """ + Fetch a city object based on name and city id + :param name: city object name + :param city_id: a city identifier + :return: [CityObject] with the provided name or alias belonging to the city with id city_id + """ + try: + # search by name first + with Session(self.engine) as session: + city_object = session.execute(select(Model).where(Model.name == name, Model.city_id == city_id)).first() + if city_object is not None: + return city_object[0] + # name not found, so search by alias instead + city_objects = session.execute( + select(Model).where(Model.aliases.contains(name), Model.city_id == city_id) + ).all() + for city_object in city_objects: + aliases = city_object[0].aliases.replace('{', '').replace('}', '').split(',') + for alias in aliases: + if alias == name: + # force the name as the alias + city_object[0].name = name + return city_object[0] + return None + except SQLAlchemyError as err: + logging.error('Error while fetching city object by name and city: %s', err) + raise SQLAlchemyError from err + except IndexError as err: + logging.error('Error while fetching city object by name and city, empty result %s', err) + raise IndexError from err diff --git a/repositories/simulation_results.py b/repositories/simulation_results.py new file mode 100644 index 0000000..3f6faa4 --- /dev/null +++ b/repositories/simulation_results.py @@ -0,0 +1,169 @@ +""" +Simulation results repository with database CRUD operations +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca +""" +import datetime +import logging + +from sqlalchemy import or_ +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from hub.persistence.repository import Repository +from hub.persistence.models import City +from hub.persistence.models import CityObject +from hub.persistence.models import SimulationResults as Model + + +class SimulationResults(Repository): + """ + Simulation results repository + """ + _instance = None + + def __init__(self, db_name: str, dotenv_path: str, app_env: str): + super().__init__(db_name, dotenv_path, app_env) + + def __new__(cls, db_name, dotenv_path, app_env): + """ + Implemented for a singleton pattern + """ + if cls._instance is None: + cls._instance = super(SimulationResults, cls).__new__(cls) + return cls._instance + + def insert(self, name: str, values: str, city_id=None, city_object_id=None): + """ + Inserts simulations results linked either with a city as a whole or with a city object + :param name: results name + :param values: the simulation results in json format + :param city_id: optional city id + :param city_object_id: optional city object id + :return: Identity id + """ + if city_id is not None: + _ = self._get_city(city_id) + else: + _ = self._get_city_object(city_object_id) + try: + simulation_result = Model(name=name, + values=values, + city_id=city_id, + city_object_id=city_object_id) + with Session(self.engine) as session: + session.add(simulation_result) + session.flush() + session.commit() + session.refresh(simulation_result) + return simulation_result.id + except SQLAlchemyError as err: + logging.error('An error occurred while creating city_object %s', err) + raise SQLAlchemyError from err + + def update(self, name: str, values: str, city_id=None, city_object_id=None): + """ + Updates simulation results for a city or a city object + :param name: The simulation results tool and workflow name + :param values: the simulation results in json format + :param city_id: optional city id + :param city_object_id: optional city object id + :return: None + """ + try: + with Session(self.engine) as session: + if city_id is not None: + session.query(Model).filter(Model.name == name, Model.city_id == city_id).update( + { + 'values': values, + 'updated': datetime.datetime.utcnow() + }) + session.commit() + elif city_object_id is not None: + session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).update( + { + 'values': values, + 'updated': datetime.datetime.utcnow() + }) + session.commit() + else: + raise NotImplementedError('Missing either city_id or city_object_id') + except SQLAlchemyError as err: + logging.error('Error while updating city object %s', err) + raise SQLAlchemyError from err + + def delete(self, name: str, city_id=None, city_object_id=None): + """ + Deletes an application with the application_uuid + :param name: The simulation results tool and workflow name + :param city_id: The id for the city owning the simulation results + :param city_object_id: the id for the city_object owning these simulation results + :return: None + """ + try: + with Session(self.engine) as session: + if city_id is not None: + session.query(Model).filter(Model.name == name, Model.city_id == city_id).delete() + session.commit() + elif city_object_id is not None: + session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).delete() + session.commit() + else: + raise NotImplementedError('Missing either city_id or city_object_id') + except SQLAlchemyError as err: + logging.error('Error while deleting application: %s', err) + raise SQLAlchemyError from err + + def _get_city(self, city_id) -> City: + """ + Fetch a city object based city id + :param city_id: a city identifier + :return: [City] with the provided city_id + """ + try: + with Session(self.engine) as session: + return session.execute(select(City).where(City.id == city_id)).first() + except SQLAlchemyError as err: + logging.error('Error while fetching city by city_id: %s', err) + raise SQLAlchemyError from err + + def _get_city_object(self, city_object_id) -> [CityObject]: + """ + Fetch a city object based city id + :param city_object_id: a city object identifier + :return: [CityObject] with the provided city_object_id + """ + try: + with Session(self.engine) as session: + return session.execute(select(CityObject).where(CityObject.id == city_object_id)).first() + except SQLAlchemyError as err: + logging.error('Error while fetching city by city_id: %s', err) + raise SQLAlchemyError from err + + def get_simulation_results_by_city_id_city_object_id_and_names(self, city_id, city_object_id, result_names=None) -> [Model]: + """ + Fetch the simulation results based in the city_id or city_object_id with the given names or all + :param city_id: the city id + :param city_object_id: the city object id + :param result_names: if given filter the results + :return: [SimulationResult] + """ + try: + with Session(self.engine) as session: + result_set = session.execute(select(Model).where(or_( + Model.city_id == city_id, + Model.city_object_id == city_object_id + ))) + results = [r[0] for r in result_set] + if not result_names: + return results + filtered_results = [] + for result in results: + if result.name in result_names: + filtered_results.append(result) + return filtered_results + except SQLAlchemyError as err: + logging.error('Error while fetching city by city_id: %s', err) + raise SQLAlchemyError from err diff --git a/repositories/user.py b/repositories/user.py new file mode 100644 index 0000000..4467f7c --- /dev/null +++ b/repositories/user.py @@ -0,0 +1,159 @@ +""" +User repository with database CRUD operations +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Peter Yefi peteryefi@gmail.com +""" +import datetime +import logging + +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session + +from hub.helpers.auth import Auth +from hub.persistence.repository import Repository +from hub.persistence.models import User as Model, Application as ApplicationModel, UserRoles + + +class User(Repository): + """ + User class + """ + _instance = None + + def __init__(self, db_name: str, dotenv_path: str, app_env: str): + super().__init__(db_name, dotenv_path, app_env) + + def __new__(cls, db_name, dotenv_path, app_env): + """ + Implemented for a singleton pattern + """ + if cls._instance is None: + cls._instance = super(User, cls).__new__(cls) + return cls._instance + + def insert(self, name: str, password: str, role: UserRoles, application_id: int): + """ + Inserts a new user + :param name: username + :param password: user password + :param role: user rol [Admin or Hub_Reader] + :param application_id: user application id + :return: Identity id + """ + try: + user = self.get_by_name_and_application(name, application_id) + if user is not None: + raise SQLAlchemyError(f'A user named {user.name} already exists for that application') + except TypeError: + pass + try: + user = Model(name=name, password=Auth.hash_password(password), role=role, application_id=application_id) + with Session(self.engine) as session: + session.add(user) + session.flush() + session.commit() + session.refresh(user) + return user.id + except SQLAlchemyError as err: + logging.error('An error occurred while creating user %s', err) + raise SQLAlchemyError from err + + def update(self, user_id: int, name: str, password: str, role: UserRoles): + """ + Updates a user + :param user_id: the id of the user to be updated + :param name: the name of the user + :param password: the password of the user + :param role: the role of the user + :return: None + """ + try: + with Session(self.engine) as session: + session.query(Model).filter(Model.id == user_id).update({ + 'name': name, + 'password': Auth.hash_password(password), + 'role': role, + 'updated': datetime.datetime.utcnow() + }) + session.commit() + except SQLAlchemyError as err: + logging.error('Error while updating user: %s', err) + raise SQLAlchemyError from err + + def delete(self, user_id: int): + """ + Deletes a user with the id + :param user_id: the user id + :return: None + """ + try: + with Session(self.engine) as session: + session.query(Model).filter(Model.id == user_id).delete() + session.commit() + except SQLAlchemyError as err: + logging.error('Error while fetching user: %s', err) + raise SQLAlchemyError from err + + def get_by_name_and_application(self, name: str, application_id: int) -> Model: + """ + Fetch user based on the email address + :param name: Username + :param application_id: User application name + :return: User matching the search criteria or None + """ + try: + with Session(self.engine) as session: + user = session.execute( + select(Model).where(Model.name == name, Model.application_id == application_id) + ).first() + session.commit() + return user[0] + except SQLAlchemyError as err: + logging.error('Error while fetching user by name and application: %s', err) + raise SQLAlchemyError from err + except TypeError as err: + logging.error('Error while fetching user, empty result %s', err) + raise TypeError from err + + def get_by_name_application_id_and_password(self, name: str, password: str, application_id: int) -> Model: + """ + Fetch user based on the name, password and application id + :param name: Username + :param password: User password + :param application_id: Application id + :return: User + """ + try: + with Session(self.engine) as session: + user = session.execute( + select(Model).where(Model.name == name, Model.application_id == application_id) + ).first() + if user: + if Auth.check_password(password, user[0].password): + return user[0] + except SQLAlchemyError as err: + logging.error('Error while fetching user by name: %s', err) + raise SQLAlchemyError from err + raise ValueError('Unauthorized') + + def get_by_name_application_uuid_and_password(self, name: str, password: str, application_uuid: str) -> Model: + """ + Fetch user based on the email and password + :param name: Username + :param password: User password + :param application_uuid: Application uuid + :return: User + """ + try: + with Session(self.engine) as session: + application = session.execute( + select(ApplicationModel).where(ApplicationModel.application_uuid == application_uuid) + ).first() + return self.get_by_name_application_id_and_password(name, password, application[0].id) + except SQLAlchemyError as err: + logging.error('Error while fetching user by name: %s', err) + raise SQLAlchemyError from err + except ValueError as err: + raise ValueError from err diff --git a/repository.py b/repository.py new file mode 100644 index 0000000..5a3b4e2 --- /dev/null +++ b/repository.py @@ -0,0 +1,22 @@ +""" +Base repository class to establish db connection +SPDX - License - Identifier: LGPL - 3.0 - or -later +Copyright © 2022 Concordia CERC group +Project Coder Peter Yefi peteryefi@gmail.com +""" +import logging +from sqlalchemy import create_engine +from hub.persistence.configuration import Configuration + + +class Repository: + """ + Base repository class to establish db connection + """ + + def __init__(self, db_name, dotenv_path: str, app_env='TEST'): + try: + self.configuration = Configuration(db_name, dotenv_path, app_env) + self.engine = create_engine(self.configuration.connection_string) + except ValueError as err: + logging.error('Missing value for credentials: %s', err)