From 31a4893ed864a482a3ac8ae13b0ce83b5d3960e3 Mon Sep 17 00:00:00 2001 From: guille Date: Fri, 13 Oct 2023 08:57:42 +0200 Subject: [PATCH] Corrections in persistence, distribution_systems_electrical_consumption property seems to be bugged --- hub/city_model_structure/building.py | 6 +- hub/persistence/db_control.py | 4 - hub/persistence/models/application.py | 3 +- hub/persistence/models/city.py | 4 +- hub/persistence/models/city_object.py | 2 +- hub/persistence/models/simulation_results.py | 4 +- hub/persistence/repositories/application.py | 35 ++++---- hub/persistence/repositories/city.py | 58 +++++++------ hub/persistence/repositories/city_object.py | 54 ++++++------ .../repositories/simulation_results.py | 85 ++++++++++--------- hub/persistence/repositories/user.py | 62 ++++++++------ hub/persistence/repository.py | 2 - tests/test_db_factory.py | 8 +- 13 files changed, 174 insertions(+), 153 deletions(-) diff --git a/hub/city_model_structure/building.py b/hub/city_model_structure/building.py index bdb8a7e3..d7efb9f2 100644 --- a/hub/city_model_structure/building.py +++ b/hub/city_model_structure/building.py @@ -743,8 +743,10 @@ class Building(CityObject): for key, item in self._distribution_systems_electrical_consumption.items(): for i in range(0, len(item)): - self._distribution_systems_electrical_consumption[key][i] += _peak_load * _consumption_fix_flow \ - * _working_hours[key] * cte.WATTS_HOUR_TO_JULES + self._distribution_systems_electrical_consumption[key][i] += ( + _peak_load * _consumption_fix_flow * _working_hours[key] * cte.WATTS_HOUR_TO_JULES + ) + return self._distribution_systems_electrical_consumption def _calculate_consumption(self, consumption_type, demand): diff --git a/hub/persistence/db_control.py b/hub/persistence/db_control.py index 5bc7df35..f53d0d30 100644 --- a/hub/persistence/db_control.py +++ b/hub/persistence/db_control.py @@ -114,10 +114,7 @@ class DBControl: result_names = [] results = {} for scenario in request_values['scenarios']: - print('scenario', scenario, results) for scenario_name in scenario.keys(): - print('scenario name', scenario_name) - result_sets = self._city.get_by_user_id_application_id_and_scenario( user_id, application_id, @@ -143,7 +140,6 @@ class DBControl: values = json.loads(value.values) values["building"] = building_name results[scenario_name].append(values) - print(scenario, results) return results def persist_city(self, city: City, pickle_path, scenario, application_id: int, user_id: int): diff --git a/hub/persistence/models/application.py b/hub/persistence/models/application.py index 3da947f4..23f05a27 100644 --- a/hub/persistence/models/application.py +++ b/hub/persistence/models/application.py @@ -7,9 +7,10 @@ Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca import datetime -from sqlalchemy.dialects.postgresql import UUID from sqlalchemy import Column, Integer, String, Sequence from sqlalchemy import DateTime +from sqlalchemy.dialects.postgresql import UUID + from hub.persistence.configuration import Models diff --git a/hub/persistence/models/city.py b/hub/persistence/models/city.py index e77e51de..35f529b1 100644 --- a/hub/persistence/models/city.py +++ b/hub/persistence/models/city.py @@ -21,8 +21,8 @@ class City(Models): pickle_path = Column(String, nullable=False) name = Column(String, nullable=False) scenario = Column(String, nullable=False) - application_id = Column(Integer, ForeignKey('application.id'), nullable=False) - user_id = Column(Integer, ForeignKey('user.id'), nullable=True) + application_id = Column(Integer, ForeignKey('application.id', ondelete='CASCADE'), nullable=False) + user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE'), nullable=True) hub_release = Column(String, nullable=False) created = Column(DateTime, default=datetime.datetime.utcnow) updated = Column(DateTime, default=datetime.datetime.utcnow) diff --git a/hub/persistence/models/city_object.py b/hub/persistence/models/city_object.py index 48715b61..1d88b9ec 100644 --- a/hub/persistence/models/city_object.py +++ b/hub/persistence/models/city_object.py @@ -21,7 +21,7 @@ class CityObject(Models): """ __tablename__ = 'city_object' id = Column(Integer, Sequence('city_object_id_seq'), primary_key=True) - city_id = Column(Integer, ForeignKey('city.id'), nullable=False) + city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=False) name = Column(String, nullable=False) aliases = Column(String, nullable=True) type = Column(String, nullable=False) diff --git a/hub/persistence/models/simulation_results.py b/hub/persistence/models/simulation_results.py index d4dea88d..a40c077e 100644 --- a/hub/persistence/models/simulation_results.py +++ b/hub/persistence/models/simulation_results.py @@ -19,8 +19,8 @@ class SimulationResults(Models): """ __tablename__ = 'simulation_results' id = Column(Integer, Sequence('simulation_results_id_seq'), primary_key=True) - city_id = Column(Integer, ForeignKey('city.id'), nullable=True) - city_object_id = Column(Integer, ForeignKey('city_object.id'), nullable=True) + city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=True) + city_object_id = Column(Integer, ForeignKey('city_object.id', ondelete='CASCADE'), nullable=True) name = Column(String, nullable=False) values = Column(JSONB, nullable=False) created = Column(DateTime, default=datetime.datetime.utcnow) diff --git a/hub/persistence/repositories/application.py b/hub/persistence/repositories/application.py index f68fd0bb..a70fde37 100644 --- a/hub/persistence/repositories/application.py +++ b/hub/persistence/repositories/application.py @@ -10,6 +10,7 @@ import logging from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm.session import Session from hub.persistence.repository import Repository from hub.persistence.models import Application as Model @@ -48,10 +49,11 @@ class Application(Repository): pass try: application = Model(name=name, description=description, application_uuid=application_uuid) - self.session.add(application) - self.session.commit() - self.session.refresh(application) - return application.id + with Session(self.engine) as session: + session.add(application) + session.commit() + session.refresh(application) + return application.id except SQLAlchemyError as err: logging.error('An error occurred while creating application %s', err) raise SQLAlchemyError from err @@ -65,10 +67,11 @@ class Application(Repository): :return: None """ try: - self.session.query(Model).filter( - Model.application_uuid == application_uuid - ).update({'name': name, 'description': description, 'updated': datetime.datetime.utcnow()}) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter( + Model.application_uuid == application_uuid + ).update({'name': name, 'description': description, 'updated': datetime.datetime.utcnow()}) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating application %s', err) raise SQLAlchemyError from err @@ -80,9 +83,10 @@ class Application(Repository): :return: None """ try: - self.session.query(Model).filter(Model.application_uuid == application_uuid).delete() - self.session.flush() - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.application_uuid == application_uuid).delete() + session.flush() + session.commit() except SQLAlchemyError as err: logging.error('Error while deleting application %s', err) raise SQLAlchemyError from err @@ -94,10 +98,11 @@ class Application(Repository): :return: Application with the provided application_uuid """ try: - result_set = self.session.execute(select(Model).where( - Model.application_uuid == application_uuid) - ).first() - return result_set[0] + with Session(self.engine) as session: + result_set = session.execute(select(Model).where( + Model.application_uuid == application_uuid) + ).first() + return result_set[0] except SQLAlchemyError as err: logging.error('Error while fetching application by application_uuid %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repositories/city.py b/hub/persistence/repositories/city.py index 9c20346f..d1e07343 100644 --- a/hub/persistence/repositories/city.py +++ b/hub/persistence/repositories/city.py @@ -54,18 +54,18 @@ class City(Repository): application_id, user_id, __version__) - - self.session.add(db_city) - self.session.flush() - self.session.commit() - for building in city.buildings: - db_city_object = CityObject(db_city.id, - building) - self.session.add(db_city_object) - self.session.flush() - self.session.commit() - self.session.refresh(db_city) - return db_city.id + with Session(self.engine) as session: + session.add(db_city) + session.flush() + session.commit() + for building in city.buildings: + db_city_object = CityObject(db_city.id, + building) + session.add(db_city_object) + session.flush() + session.commit() + session.refresh(db_city) + return db_city.id except SQLAlchemyError as err: logging.error('An error occurred while creating a city %s', err) raise SQLAlchemyError from err @@ -79,8 +79,9 @@ class City(Repository): """ try: now = datetime.datetime.utcnow() - self.session.query(Model).filter(Model.id == city_id).update({'name': city.name, 'updated': now}) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.id == city_id).update({'name': city.name, 'updated': now}) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating city %s', err) raise SQLAlchemyError from err @@ -92,9 +93,10 @@ class City(Repository): :return: None """ try: - self.session.query(CityObject).filter(CityObject.city_id == city_id).delete() - self.session.query(Model).filter(Model.id == city_id).delete() - self.session.commit() + with Session(self.engine) as session: + session.query(CityObject).filter(CityObject.city_id == city_id).delete() + session.query(Model).filter(Model.id == city_id).delete() + session.commit() except SQLAlchemyError as err: logging.error('Error while fetching city %s', err) raise SQLAlchemyError from err @@ -108,13 +110,12 @@ class City(Repository): :return: [ModelCity] """ try: - result_set = self.session.execute(select(Model).where(Model.user_id == user_id, - Model.application_id == application_id, - Model.scenario == scenario - )).all() - self.session.close() - self.session = Session(self.engine) - return result_set + with Session(self.engine) as session: + result_set = session.execute(select(Model).where(Model.user_id == user_id, + Model.application_id == application_id, + Model.scenario == scenario + )).all() + return result_set except SQLAlchemyError as err: logging.error('Error while fetching city by name %s', err) raise SQLAlchemyError from err @@ -127,10 +128,11 @@ class City(Repository): :return: ModelCity """ try: - result_set = self.session.execute( - select(Model).where(Model.user_id == user_id, Model.application_id == application_id) - ) - return [r[0] for r in result_set] + with Session(self.engine) as session: + result_set = session.execute( + select(Model).where(Model.user_id == user_id, Model.application_id == application_id) + ) + return [r[0] for r in result_set] except SQLAlchemyError as err: logging.error('Error while fetching city by name %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repositories/city_object.py b/hub/persistence/repositories/city_object.py index aa69a0e3..30287411 100644 --- a/hub/persistence/repositories/city_object.py +++ b/hub/persistence/repositories/city_object.py @@ -46,10 +46,11 @@ class CityObject(Repository): try: city_object = Model(city_id=city_id, building=building) - self.session.add(city_object) - self.session.flush() - self.session.commit() - self.session.refresh(city_object) + with Session(self.engine) as session: + session.add(city_object) + session.flush() + session.commit() + session.refresh(city_object) return city_object.id except SQLAlchemyError as err: logging.error('An error occurred while creating city_object %s', err) @@ -68,17 +69,18 @@ class CityObject(Repository): for usage in internal_zone.usages: object_usage = f'{object_usage}{usage.name}_{usage.percentage} ' object_usage = object_usage.rstrip() - self.session.query(Model).filter(Model.name == building.name, Model.city_id == city_id).update( - {'name': building.name, - 'alias': building.alias, - 'object_type': building.type, - 'year_of_construction': building.year_of_construction, - 'function': building.function, - 'usage': object_usage, - 'volume': building.volume, - 'area': building.floor_area, - 'updated': datetime.datetime.utcnow()}) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.name == building.name, Model.city_id == city_id).update( + {'name': building.name, + 'alias': building.alias, + 'object_type': building.type, + 'year_of_construction': building.year_of_construction, + 'function': building.function, + 'usage': object_usage, + 'volume': building.volume, + 'area': building.floor_area, + 'updated': datetime.datetime.utcnow()}) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating city object %s', err) raise SQLAlchemyError from err @@ -91,8 +93,9 @@ class CityObject(Repository): :return: None """ try: - self.session.query(Model).filter(Model.city_id == city_id, Model.name == name).delete() - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.city_id == city_id, Model.name == name).delete() + session.commit() except SQLAlchemyError as err: logging.error('Error while deleting application %s', err) raise SQLAlchemyError from err @@ -106,15 +109,14 @@ class CityObject(Repository): """ try: # search by name first - city_object = self.session.execute(select(Model).where(Model.name == name, Model.city_id == city_id)).first() - if city_object is not None: - return city_object[0] - # name not found, so search by alias instead - city_objects = self.session.execute( - select(Model).where(Model.aliases.contains(name), Model.city_id == city_id) - ).all() - self.session.close() - self.session = Session(self.engine) + with Session(self.engine) as session: + city_object = session.execute(select(Model).where(Model.name == name, Model.city_id == city_id)).first() + if city_object is not None: + return city_object[0] + # name not found, so search by alias instead + city_objects = session.execute( + select(Model).where(Model.aliases.contains(name), Model.city_id == city_id) + ).all() for city_object in city_objects: aliases = city_object[0].aliases.replace('{', '').replace('}', '').split(',') for alias in aliases: diff --git a/hub/persistence/repositories/simulation_results.py b/hub/persistence/repositories/simulation_results.py index 0147eafc..3f6faa48 100644 --- a/hub/persistence/repositories/simulation_results.py +++ b/hub/persistence/repositories/simulation_results.py @@ -10,6 +10,7 @@ import logging from sqlalchemy import or_ from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session from hub.persistence.repository import Repository from hub.persistence.models import City @@ -52,11 +53,12 @@ class SimulationResults(Repository): values=values, city_id=city_id, city_object_id=city_object_id) - self.session.add(simulation_result) - self.session.flush() - self.session.commit() - self.session.refresh(simulation_result) - return simulation_result.id + with Session(self.engine) as session: + session.add(simulation_result) + session.flush() + session.commit() + session.refresh(simulation_result) + return simulation_result.id except SQLAlchemyError as err: logging.error('An error occurred while creating city_object %s', err) raise SQLAlchemyError from err @@ -71,22 +73,23 @@ class SimulationResults(Repository): :return: None """ try: - if city_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_id == city_id).update( + with Session(self.engine) as session: + if city_id is not None: + session.query(Model).filter(Model.name == name, Model.city_id == city_id).update( { 'values': values, 'updated': datetime.datetime.utcnow() }) - self.session.commit() - elif city_object_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).update( - { - 'values': values, - 'updated': datetime.datetime.utcnow() - }) - self.session.commit() - else: - raise NotImplementedError('Missing either city_id or city_object_id') + session.commit() + elif city_object_id is not None: + session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).update( + { + 'values': values, + 'updated': datetime.datetime.utcnow() + }) + session.commit() + else: + raise NotImplementedError('Missing either city_id or city_object_id') except SQLAlchemyError as err: logging.error('Error while updating city object %s', err) raise SQLAlchemyError from err @@ -100,14 +103,15 @@ class SimulationResults(Repository): :return: None """ try: - if city_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_id == city_id).delete() - self.session.commit() - elif city_object_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).delete() - self.session.commit() - else: - raise NotImplementedError('Missing either city_id or city_object_id') + with Session(self.engine) as session: + if city_id is not None: + session.query(Model).filter(Model.name == name, Model.city_id == city_id).delete() + session.commit() + elif city_object_id is not None: + session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).delete() + session.commit() + else: + raise NotImplementedError('Missing either city_id or city_object_id') except SQLAlchemyError as err: logging.error('Error while deleting application: %s', err) raise SQLAlchemyError from err @@ -119,7 +123,8 @@ class SimulationResults(Repository): :return: [City] with the provided city_id """ try: - return self.session.execute(select(City).where(City.id == city_id)).first() + with Session(self.engine) as session: + return session.execute(select(City).where(City.id == city_id)).first() except SQLAlchemyError as err: logging.error('Error while fetching city by city_id: %s', err) raise SQLAlchemyError from err @@ -131,7 +136,8 @@ class SimulationResults(Repository): :return: [CityObject] with the provided city_object_id """ try: - return self.session.execute(select(CityObject).where(CityObject.id == city_object_id)).first() + with Session(self.engine) as session: + return session.execute(select(CityObject).where(CityObject.id == city_object_id)).first() except SQLAlchemyError as err: logging.error('Error while fetching city by city_id: %s', err) raise SQLAlchemyError from err @@ -145,18 +151,19 @@ class SimulationResults(Repository): :return: [SimulationResult] """ try: - result_set = self.session.execute(select(Model).where(or_( - Model.city_id == city_id, - Model.city_object_id == city_object_id - ))) - results = [r[0] for r in result_set] - if not result_names: - return results - filtered_results = [] - for result in results: - if result.name in result_names: - filtered_results.append(result) - return filtered_results + with Session(self.engine) as session: + result_set = session.execute(select(Model).where(or_( + Model.city_id == city_id, + Model.city_object_id == city_object_id + ))) + results = [r[0] for r in result_set] + if not result_names: + return results + filtered_results = [] + for result in results: + if result.name in result_names: + filtered_results.append(result) + return filtered_results except SQLAlchemyError as err: logging.error('Error while fetching city by city_id: %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repositories/user.py b/hub/persistence/repositories/user.py index 51101858..4467f7cb 100644 --- a/hub/persistence/repositories/user.py +++ b/hub/persistence/repositories/user.py @@ -9,6 +9,7 @@ import logging from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session from hub.helpers.auth import Auth from hub.persistence.repository import Repository @@ -49,10 +50,11 @@ class User(Repository): pass try: user = Model(name=name, password=Auth.hash_password(password), role=role, application_id=application_id) - self.session.add(user) - self.session.flush() - self.session.commit() - self.session.refresh(user) + with Session(self.engine) as session: + session.add(user) + session.flush() + session.commit() + session.refresh(user) return user.id except SQLAlchemyError as err: logging.error('An error occurred while creating user %s', err) @@ -68,13 +70,14 @@ class User(Repository): :return: None """ try: - self.session.query(Model).filter(Model.id == user_id).update({ - 'name': name, - 'password': Auth.hash_password(password), - 'role': role, - 'updated': datetime.datetime.utcnow() - }) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.id == user_id).update({ + 'name': name, + 'password': Auth.hash_password(password), + 'role': role, + 'updated': datetime.datetime.utcnow() + }) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating user: %s', err) raise SQLAlchemyError from err @@ -86,8 +89,9 @@ class User(Repository): :return: None """ try: - self.session.query(Model).filter(Model.id == user_id).delete() - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.id == user_id).delete() + session.commit() except SQLAlchemyError as err: logging.error('Error while fetching user: %s', err) raise SQLAlchemyError from err @@ -100,10 +104,12 @@ class User(Repository): :return: User matching the search criteria or None """ try: - user = self.session.execute( - select(Model).where(Model.name == name, Model.application_id == application_id) - ).first() - return user[0] + with Session(self.engine) as session: + user = session.execute( + select(Model).where(Model.name == name, Model.application_id == application_id) + ).first() + session.commit() + return user[0] except SQLAlchemyError as err: logging.error('Error while fetching user by name and application: %s', err) raise SQLAlchemyError from err @@ -120,12 +126,13 @@ class User(Repository): :return: User """ try: - user = self.session.execute( - select(Model).where(Model.name == name, Model.application_id == application_id) - ).first() - if user: - if Auth.check_password(password, user[0].password): - return user[0] + with Session(self.engine) as session: + user = session.execute( + select(Model).where(Model.name == name, Model.application_id == application_id) + ).first() + if user: + if Auth.check_password(password, user[0].password): + return user[0] except SQLAlchemyError as err: logging.error('Error while fetching user by name: %s', err) raise SQLAlchemyError from err @@ -140,10 +147,11 @@ class User(Repository): :return: User """ try: - application = self.session.execute( - select(ApplicationModel).where(ApplicationModel.application_uuid == application_uuid) - ).first() - return self.get_by_name_application_id_and_password(name, password, application[0].id) + with Session(self.engine) as session: + application = session.execute( + select(ApplicationModel).where(ApplicationModel.application_uuid == application_uuid) + ).first() + return self.get_by_name_application_id_and_password(name, password, application[0].id) except SQLAlchemyError as err: logging.error('Error while fetching user by name: %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repository.py b/hub/persistence/repository.py index 97ed9f91..5a3b4e26 100644 --- a/hub/persistence/repository.py +++ b/hub/persistence/repository.py @@ -6,7 +6,6 @@ Project Coder Peter Yefi peteryefi@gmail.com """ import logging from sqlalchemy import create_engine -from sqlalchemy.orm import Session from hub.persistence.configuration import Configuration @@ -19,6 +18,5 @@ class Repository: try: self.configuration = Configuration(db_name, dotenv_path, app_env) self.engine = create_engine(self.configuration.connection_string) - self.session = Session(self.engine) except ValueError as err: logging.error('Missing value for credentials: %s', err) diff --git a/tests/test_db_factory.py b/tests/test_db_factory.py index 94b599a7..4c216ad2 100644 --- a/tests/test_db_factory.py +++ b/tests/test_db_factory.py @@ -103,16 +103,16 @@ class Control: app_env='TEST', dotenv_path=dotenv_path) - self._application_uuid = '60b7fc1b-f389-4254-9ffd-22a4cf32c7a3' + self._application_uuid = 'b9e0ce80-1218-410c-8a64-9d9b7026aad8' self._application_id = 1 self._user_id = 1 self._application_id = self._database.persist_application( - 'City_layers', - 'City layers test user', + 'test', + 'test', self.application_uuid ) - self._user_id = self._database.create_user('city_layers', self._application_id, 'city_layers', UserRoles.Admin) + self._user_id = self._database.create_user('test', self._application_id, 'test', UserRoles.Admin) self._pickle_path = Path('tests_data/pickle_path.bz2').resolve()