From 6b24a59178b263a3a033710c3bd03dacad7a236d Mon Sep 17 00:00:00 2001 From: guille Date: Tue, 10 Oct 2023 11:16:26 +0200 Subject: [PATCH 1/8] Add exports for cesium tileset and glb format --- hub/exports/exports_factory.py | 28 ++++- hub/exports/formats/cesiumjs_tileset.py | 137 ++++++++++++++++++++++++ hub/exports/formats/glb.py | 52 +++++++++ hub/exports/formats/obj.py | 2 +- tests/test_exports.py | 24 ++++- 5 files changed, 236 insertions(+), 7 deletions(-) create mode 100644 hub/exports/formats/cesiumjs_tileset.py create mode 100644 hub/exports/formats/glb.py diff --git a/hub/exports/exports_factory.py b/hub/exports/exports_factory.py index 4b7216ea..7a63c1da 100644 --- a/hub/exports/exports_factory.py +++ b/hub/exports/exports_factory.py @@ -7,9 +7,11 @@ Project Coder Guille Gutierrez guillermo.gutierrezmorote@concordia.ca from pathlib import Path +from hub.exports.formats.glb import Glb from hub.exports.formats.obj import Obj from hub.exports.formats.simplified_radiosity_algorithm import SimplifiedRadiosityAlgorithm from hub.exports.formats.stl import Stl +from hub.exports.formats.cesiumjs_tileset import CesiumjsTileset from hub.helpers.utils import validate_import_export_type @@ -17,7 +19,7 @@ class ExportsFactory: """ Exports factory class """ - def __init__(self, handler, city, path, target_buildings=None, adjacent_buildings=None): + def __init__(self, handler, city, path, target_buildings=None, adjacent_buildings=None, base_uri=None): self._city = city self._handler = '_' + handler.lower() validate_import_export_type(ExportsFactory, handler) @@ -26,6 +28,7 @@ class ExportsFactory: self._path = path self._target_buildings = target_buildings self._adjacent_buildings = adjacent_buildings + self._base_uri = base_uri @property def _citygml(self): @@ -61,9 +64,26 @@ class ExportsFactory: Export the city to Simplified Radiosity Algorithm xml format :return: None """ - return SimplifiedRadiosityAlgorithm(self._city, - (self._path / f'{self._city.name}_sra.xml'), - target_buildings=self._target_buildings) + return SimplifiedRadiosityAlgorithm( + self._city, (self._path / f'{self._city.name}_sra.xml'), target_buildings=self._target_buildings + ) + + @property + def _cesiumjs_tileset(self): + """ + Export the city to a cesiumJs tileset format + :return: None + """ + return CesiumjsTileset( + self._city, + (self._path / f'{self._city.name}.json'), + target_buildings=self._target_buildings, + base_uri=self._base_uri + ) + + @property + def _glb(self): + return Glb(self._city, self._path, target_buildings=self._target_buildings) def export(self): """ diff --git a/hub/exports/formats/cesiumjs_tileset.py b/hub/exports/formats/cesiumjs_tileset.py new file mode 100644 index 00000000..9589a980 --- /dev/null +++ b/hub/exports/formats/cesiumjs_tileset.py @@ -0,0 +1,137 @@ +import json +import math + +import pyproj +from pyproj import Transformer + +from hub.helpers.geometry_helper import GeometryHelper + + +class CesiumjsTileset: + def __init__(self, city, file_name, target_buildings=None, base_uri=None): + self._city = city + self._file_name = file_name + self._target_buildings = target_buildings + if base_uri is None: + base_uri = '.' + self._base_uri = base_uri + try: + srs_name = self._city.srs_name + if self._city.srs_name in GeometryHelper.srs_transformations: + srs_name = GeometryHelper.srs_transformations[self._city.srs_name] + input_reference = pyproj.CRS(srs_name) # Projected coordinate system from input data + except pyproj.exceptions.CRSError as err: + raise pyproj.exceptions.CRSError from err + self._to_gps = Transformer.from_crs(input_reference, pyproj.CRS('EPSG:4326')) + self._tile_set = { + 'asset': { + 'version': '1.1', + "tilesetVersion": "1.2.3" + }, + 'schema': { + 'id': "building", + 'classes': { + 'building': { + "properties": { + 'name': { + 'type': 'STRING' + }, + 'position': { + 'type': 'SCALAR', + 'array': True, + 'componentType': 'FLOAT32' + }, + 'aliases': { + 'type': 'STRING', + 'array': True, + }, + 'volume': { + 'type': 'SCALAR', + 'componentType': 'FLOAT32' + }, + 'floor_area': { + 'type': 'SCALAR', + 'componentType': 'FLOAT32' + }, + 'max_height': { + 'type': 'SCALAR', + 'componentType': 'INT32' + }, + 'year_of_construction': { + 'type': 'SCALAR', + 'componentType': 'INT32' + }, + 'function': { + 'type': 'STRING' + }, + 'usages_percentage': { + 'type': 'STRING' + } + } + } + } + }, + 'geometricError': 240, + 'root': { + 'boundingVolume': { + 'box': CesiumjsTileset._box_values(self._city.upper_corner, self._city.lower_corner) + }, + 'geometricError': 70, + 'refine': 'ADD', + 'children': [] + } + } + + self._export() + + @staticmethod + def _box_values(upper_corner, lower_corner): + x = (upper_corner[0] - lower_corner[0]) / 2 + y = (upper_corner[1] - lower_corner[1]) / 2 + z = (upper_corner[2] - lower_corner[2]) / 2 + return [x, y, z, x, 0, 0, 0, y, 0, 0, 0, z] + + def _ground_coordinates(self, coordinates): + ground_coordinates = [] + for coordinate in coordinates: + ground_coordinates.append( + (coordinate[0] - self._city.lower_corner[0], coordinate[1] - self._city.lower_corner[1]) + ) + return ground_coordinates + + def _export(self): + for building in self._city.buildings: + upper_corner = [-math.inf, -math.inf, 0] + lower_corner = [math.inf, math.inf, 0] + + for surface in building.grounds: # todo: maybe we should add the terrain? + coordinates = self._ground_coordinates(surface.solid_polygon.coordinates) + lower_corner = [min([c[0] for c in coordinates]), min([c[1] for c in coordinates]), 0] + upper_corner = [max([c[0] for c in coordinates]), max([c[1] for c in coordinates]), building.max_height] + tile = { + 'boundingVolume': { + 'box': CesiumjsTileset._box_values(upper_corner, lower_corner) + }, + 'geometricError': 70, + 'metadata': { + 'class': 'building', + 'properties': { + 'name': building.name, + 'position': self._to_gps.transform(lower_corner[0], lower_corner[1]), + 'aliases': building.aliases, + 'volume': building.volume, + 'floor_area': building.floor_area, + 'max_height': building.max_height, + 'year_of_construction': building.year_of_construction, + 'function': building.function, + 'usages_percentage': building.usages_percentage + } + }, + 'content': { + 'uri': f'{self._base_uri}/{building.name}.glb' + } + } + self._tile_set['root']['children'].append(tile) + + with open(self._file_name, 'w') as f: + json.dump(self._tile_set, f) diff --git a/hub/exports/formats/glb.py b/hub/exports/formats/glb.py new file mode 100644 index 00000000..558705c8 --- /dev/null +++ b/hub/exports/formats/glb.py @@ -0,0 +1,52 @@ +import glob +import os +import shutil +import subprocess + +from hub.city_model_structure.city import City +from hub.exports.formats.obj import Obj + + +class GltExceptionError(Exception): + """ + Glt execution error + """ + + +class Glb: + def __init__(self, city, path, target_buildings=None): + self._city = city + self._path = path + if target_buildings is None: + target_buildings = [b.name for b in self._city.buildings] + self._target_buildings = target_buildings + self._export() + + @property + def _obj2gtl(self): + """ + Get the SRA installation path + :return: str + """ + return shutil.which('obj2gltf') + + def _export(self): + try: + for building in self._city.buildings: + city = City(building.lower_corner, building.upper_corner, self._city.srs_name) + city.name = building.name + city.add_city_object(building) + Obj( city, self._path) + glb = f'{self._path}/{building.name}.glb' + subprocess.run([ + self._obj2gtl, + '-i', f'{self._path}/{building.name}.obj', + '-o', f'{glb}', + '-b', + '--normalTexture', f'{self._path}/{building.name}.mtl' + ]) + os.unlink(f'{self._path}/{building.name}.obj') + os.unlink(f'{self._path}/{building.name}.mtl') + except (subprocess.SubprocessError, subprocess.TimeoutExpired, subprocess.CalledProcessError) as err: + raise GltExceptionError from err + diff --git a/hub/exports/formats/obj.py b/hub/exports/formats/obj.py index 5faa02d3..219bb1ac 100644 --- a/hub/exports/formats/obj.py +++ b/hub/exports/formats/obj.py @@ -55,7 +55,7 @@ class Obj: with open(mtl_file_path, 'w', encoding='utf-8') as mtl: mtl.write("newmtl cerc_base_material\n") mtl.write("Ka 1.0 1.0 1.0 # Ambient color (white)\n") - mtl.write("Kd 0.3 0.8 0.3 # Diffuse color (greenish)\n") + mtl.write("Kd 0.3 0.1 0.3 # Diffuse color (greenish)\n") mtl.write("Ks 1.0 1.0 1.0 # Specular color (white)\n") mtl.write("Ns 400.0 # Specular exponent (defines shininess)\n") vertices = {} diff --git a/tests/test_exports.py b/tests/test_exports.py index ff808968..a617e67d 100644 --- a/tests/test_exports.py +++ b/tests/test_exports.py @@ -5,7 +5,7 @@ Copyright © 2022 Concordia CERC group Project Coder Guille Gutierrez guillermo.gutierrezmorote@concordia.ca Code contributors: Pilar Monsalvete Alvarez de Uribarri pilar.monsalvete@concordia.ca """ - +import json import logging.handlers from pathlib import Path from unittest import TestCase @@ -66,7 +66,7 @@ class TestExports(TestCase): def _export(self, export_type, from_pickle=False): self._complete_city = self._get_complete_city(from_pickle) - ExportsFactory(export_type, self._complete_city, self._output_path).export() + ExportsFactory(export_type, self._complete_city, self._output_path, base_uri='../glb').export() def _export_building_energy(self, export_type, from_pickle=False): self._complete_city = self._get_complete_city(from_pickle) @@ -78,6 +78,26 @@ class TestExports(TestCase): """ self._export('obj', False) + def test_cesiumjs_tileset_export(self): + """ + export to cesiumjs tileset + """ + self._export('cesiumjs_tileset', False) + tileset = Path(self._output_path / f'{self._city.name}.json') + self.assertTrue(tileset.exists()) + with open(tileset, 'r') as f: + json_tileset = json.load(f) + self.assertEqual(1, len(json_tileset['root']['children']), "Wrong number of children") + + def test_glb_export(self): + """ + export to glb format + """ + self._export('glb', False) + for building in self._city.buildings: + glb_file = Path(self._output_path / f'{building.name}.glb') + self.assertTrue(glb_file.exists(), f'{building.name} Building glb wasn\'t correctly generated') + def test_energy_ade_export(self): """ export to energy ADE From d6bfe730ebdb7a23a241086954f40560a7450ec7 Mon Sep 17 00:00:00 2001 From: guille Date: Wed, 11 Oct 2023 06:00:34 +0200 Subject: [PATCH 2/8] add non-standard render position to tileset --- hub/exports/formats/cesiumjs_tileset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hub/exports/formats/cesiumjs_tileset.py b/hub/exports/formats/cesiumjs_tileset.py index 9589a980..50018f5a 100644 --- a/hub/exports/formats/cesiumjs_tileset.py +++ b/hub/exports/formats/cesiumjs_tileset.py @@ -28,6 +28,7 @@ class CesiumjsTileset: 'version': '1.1', "tilesetVersion": "1.2.3" }, + 'position': self._to_gps.transform(self._city.lower_corner[0], self._city.lower_corner[1]), 'schema': { 'id': "building", 'classes': { @@ -108,6 +109,7 @@ class CesiumjsTileset: coordinates = self._ground_coordinates(surface.solid_polygon.coordinates) lower_corner = [min([c[0] for c in coordinates]), min([c[1] for c in coordinates]), 0] upper_corner = [max([c[0] for c in coordinates]), max([c[1] for c in coordinates]), building.max_height] + tile = { 'boundingVolume': { 'box': CesiumjsTileset._box_values(upper_corner, lower_corner) From 3578d2faae47a0029a01edd77fa42e9a085f52f0 Mon Sep 17 00:00:00 2001 From: guille Date: Wed, 11 Oct 2023 06:01:15 +0200 Subject: [PATCH 3/8] add temporary indentation to tilesets --- hub/exports/formats/cesiumjs_tileset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/exports/formats/cesiumjs_tileset.py b/hub/exports/formats/cesiumjs_tileset.py index 50018f5a..6dd0ad24 100644 --- a/hub/exports/formats/cesiumjs_tileset.py +++ b/hub/exports/formats/cesiumjs_tileset.py @@ -136,4 +136,4 @@ class CesiumjsTileset: self._tile_set['root']['children'].append(tile) with open(self._file_name, 'w') as f: - json.dump(self._tile_set, f) + json.dump(self._tile_set, f, indent=2) From 7cb4600c154e166b65a6a625c1e3c3318ad1d9ec Mon Sep 17 00:00:00 2001 From: guille Date: Wed, 11 Oct 2023 14:32:31 +0200 Subject: [PATCH 4/8] Correct the positions for the boxes --- hub/exports/formats/cesiumjs_tileset.py | 24 ++++++++++++++++++++---- hub/exports/formats/glb.py | 7 +++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/hub/exports/formats/cesiumjs_tileset.py b/hub/exports/formats/cesiumjs_tileset.py index 6dd0ad24..99c04d3a 100644 --- a/hub/exports/formats/cesiumjs_tileset.py +++ b/hub/exports/formats/cesiumjs_tileset.py @@ -23,6 +23,14 @@ class CesiumjsTileset: except pyproj.exceptions.CRSError as err: raise pyproj.exceptions.CRSError from err self._to_gps = Transformer.from_crs(input_reference, pyproj.CRS('EPSG:4326')) + city_upper_corner = [ + self._city.upper_corner[0] - self._city.lower_corner[0], + self._city.upper_corner[1] - self._city.lower_corner[1], + self._city.upper_corner[2] - self._city.lower_corner[2] + ] + city_lower_corner = [0, 0, 0] + + print('root') self._tile_set = { 'asset': { 'version': '1.1', @@ -75,7 +83,7 @@ class CesiumjsTileset: 'geometricError': 240, 'root': { 'boundingVolume': { - 'box': CesiumjsTileset._box_values(self._city.upper_corner, self._city.lower_corner) + 'box': CesiumjsTileset._box_values(city_upper_corner, city_lower_corner) }, 'geometricError': 70, 'refine': 'ADD', @@ -87,10 +95,13 @@ class CesiumjsTileset: @staticmethod def _box_values(upper_corner, lower_corner): + x = (upper_corner[0] - lower_corner[0]) / 2 + x_center = ((upper_corner[0] - lower_corner[0]) / 2) + lower_corner[0] y = (upper_corner[1] - lower_corner[1]) / 2 + y_center = ((upper_corner[1] - lower_corner[1]) / 2) + lower_corner[1] z = (upper_corner[2] - lower_corner[2]) / 2 - return [x, y, z, x, 0, 0, 0, y, 0, 0, 0, z] + return [x_center, y_center, z, x, 0, 0, 0, y, 0, 0, 0, z] def _ground_coordinates(self, coordinates): ground_coordinates = [] @@ -104,10 +115,15 @@ class CesiumjsTileset: for building in self._city.buildings: upper_corner = [-math.inf, -math.inf, 0] lower_corner = [math.inf, math.inf, 0] - + lower_corner_coordinates = lower_corner for surface in building.grounds: # todo: maybe we should add the terrain? coordinates = self._ground_coordinates(surface.solid_polygon.coordinates) lower_corner = [min([c[0] for c in coordinates]), min([c[1] for c in coordinates]), 0] + lower_corner_coordinates = [ + min([c[0] for c in surface.solid_polygon.coordinates]), + min([c[1] for c in surface.solid_polygon.coordinates]), + 0 + ] upper_corner = [max([c[0] for c in coordinates]), max([c[1] for c in coordinates]), building.max_height] tile = { @@ -119,7 +135,7 @@ class CesiumjsTileset: 'class': 'building', 'properties': { 'name': building.name, - 'position': self._to_gps.transform(lower_corner[0], lower_corner[1]), + 'position': self._to_gps.transform(lower_corner_coordinates[0], lower_corner_coordinates[1]), 'aliases': building.aliases, 'volume': building.volume, 'floor_area': building.floor_area, diff --git a/hub/exports/formats/glb.py b/hub/exports/formats/glb.py index 558705c8..08776ccc 100644 --- a/hub/exports/formats/glb.py +++ b/hub/exports/formats/glb.py @@ -41,10 +41,13 @@ class Glb: subprocess.run([ self._obj2gtl, '-i', f'{self._path}/{building.name}.obj', - '-o', f'{glb}', '-b', - '--normalTexture', f'{self._path}/{building.name}.mtl' + '-o', f'{glb}', + '--triangleWindingOrderSanitization', + '--inputUpAxis', 'Y', + '--outputUpAxis', 'Y' ]) + os.unlink(f'{self._path}/{building.name}.obj') os.unlink(f'{self._path}/{building.name}.mtl') except (subprocess.SubprocessError, subprocess.TimeoutExpired, subprocess.CalledProcessError) as err: From fe76d2fe3db1eec4a0929985eb457a73575958d0 Mon Sep 17 00:00:00 2001 From: guille Date: Wed, 11 Oct 2023 17:16:18 +0200 Subject: [PATCH 5/8] Correct the positions for the boxes --- hub/exports/formats/cesiumjs_tileset.py | 6 ++---- hub/exports/formats/glb.py | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hub/exports/formats/cesiumjs_tileset.py b/hub/exports/formats/cesiumjs_tileset.py index 99c04d3a..2f23c99d 100644 --- a/hub/exports/formats/cesiumjs_tileset.py +++ b/hub/exports/formats/cesiumjs_tileset.py @@ -29,8 +29,6 @@ class CesiumjsTileset: self._city.upper_corner[2] - self._city.lower_corner[2] ] city_lower_corner = [0, 0, 0] - - print('root') self._tile_set = { 'asset': { 'version': '1.1', @@ -80,7 +78,7 @@ class CesiumjsTileset: } } }, - 'geometricError': 240, + 'geometricError': 500, 'root': { 'boundingVolume': { 'box': CesiumjsTileset._box_values(city_upper_corner, city_lower_corner) @@ -130,7 +128,7 @@ class CesiumjsTileset: 'boundingVolume': { 'box': CesiumjsTileset._box_values(upper_corner, lower_corner) }, - 'geometricError': 70, + 'geometricError': 250, 'metadata': { 'class': 'building', 'properties': { diff --git a/hub/exports/formats/glb.py b/hub/exports/formats/glb.py index 08776ccc..67b5a960 100644 --- a/hub/exports/formats/glb.py +++ b/hub/exports/formats/glb.py @@ -33,10 +33,11 @@ class Glb: def _export(self): try: for building in self._city.buildings: - city = City(building.lower_corner, building.upper_corner, self._city.srs_name) + print(self._city.lower_corner, self._city.upper_corner) + city = City(self._city.lower_corner, self._city.upper_corner, self._city.srs_name) city.name = building.name city.add_city_object(building) - Obj( city, self._path) + Obj(city, self._path) glb = f'{self._path}/{building.name}.glb' subprocess.run([ self._obj2gtl, From 31a4893ed864a482a3ac8ae13b0ce83b5d3960e3 Mon Sep 17 00:00:00 2001 From: guille Date: Fri, 13 Oct 2023 08:57:42 +0200 Subject: [PATCH 6/8] Corrections in persistence, distribution_systems_electrical_consumption property seems to be bugged --- hub/city_model_structure/building.py | 6 +- hub/persistence/db_control.py | 4 - hub/persistence/models/application.py | 3 +- hub/persistence/models/city.py | 4 +- hub/persistence/models/city_object.py | 2 +- hub/persistence/models/simulation_results.py | 4 +- hub/persistence/repositories/application.py | 35 ++++---- hub/persistence/repositories/city.py | 58 +++++++------ hub/persistence/repositories/city_object.py | 54 ++++++------ .../repositories/simulation_results.py | 85 ++++++++++--------- hub/persistence/repositories/user.py | 62 ++++++++------ hub/persistence/repository.py | 2 - tests/test_db_factory.py | 8 +- 13 files changed, 174 insertions(+), 153 deletions(-) diff --git a/hub/city_model_structure/building.py b/hub/city_model_structure/building.py index bdb8a7e3..d7efb9f2 100644 --- a/hub/city_model_structure/building.py +++ b/hub/city_model_structure/building.py @@ -743,8 +743,10 @@ class Building(CityObject): for key, item in self._distribution_systems_electrical_consumption.items(): for i in range(0, len(item)): - self._distribution_systems_electrical_consumption[key][i] += _peak_load * _consumption_fix_flow \ - * _working_hours[key] * cte.WATTS_HOUR_TO_JULES + self._distribution_systems_electrical_consumption[key][i] += ( + _peak_load * _consumption_fix_flow * _working_hours[key] * cte.WATTS_HOUR_TO_JULES + ) + return self._distribution_systems_electrical_consumption def _calculate_consumption(self, consumption_type, demand): diff --git a/hub/persistence/db_control.py b/hub/persistence/db_control.py index 5bc7df35..f53d0d30 100644 --- a/hub/persistence/db_control.py +++ b/hub/persistence/db_control.py @@ -114,10 +114,7 @@ class DBControl: result_names = [] results = {} for scenario in request_values['scenarios']: - print('scenario', scenario, results) for scenario_name in scenario.keys(): - print('scenario name', scenario_name) - result_sets = self._city.get_by_user_id_application_id_and_scenario( user_id, application_id, @@ -143,7 +140,6 @@ class DBControl: values = json.loads(value.values) values["building"] = building_name results[scenario_name].append(values) - print(scenario, results) return results def persist_city(self, city: City, pickle_path, scenario, application_id: int, user_id: int): diff --git a/hub/persistence/models/application.py b/hub/persistence/models/application.py index 3da947f4..23f05a27 100644 --- a/hub/persistence/models/application.py +++ b/hub/persistence/models/application.py @@ -7,9 +7,10 @@ Project Coder Guille Gutierrez Guillermo.GutierrezMorote@concordia.ca import datetime -from sqlalchemy.dialects.postgresql import UUID from sqlalchemy import Column, Integer, String, Sequence from sqlalchemy import DateTime +from sqlalchemy.dialects.postgresql import UUID + from hub.persistence.configuration import Models diff --git a/hub/persistence/models/city.py b/hub/persistence/models/city.py index e77e51de..35f529b1 100644 --- a/hub/persistence/models/city.py +++ b/hub/persistence/models/city.py @@ -21,8 +21,8 @@ class City(Models): pickle_path = Column(String, nullable=False) name = Column(String, nullable=False) scenario = Column(String, nullable=False) - application_id = Column(Integer, ForeignKey('application.id'), nullable=False) - user_id = Column(Integer, ForeignKey('user.id'), nullable=True) + application_id = Column(Integer, ForeignKey('application.id', ondelete='CASCADE'), nullable=False) + user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE'), nullable=True) hub_release = Column(String, nullable=False) created = Column(DateTime, default=datetime.datetime.utcnow) updated = Column(DateTime, default=datetime.datetime.utcnow) diff --git a/hub/persistence/models/city_object.py b/hub/persistence/models/city_object.py index 48715b61..1d88b9ec 100644 --- a/hub/persistence/models/city_object.py +++ b/hub/persistence/models/city_object.py @@ -21,7 +21,7 @@ class CityObject(Models): """ __tablename__ = 'city_object' id = Column(Integer, Sequence('city_object_id_seq'), primary_key=True) - city_id = Column(Integer, ForeignKey('city.id'), nullable=False) + city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=False) name = Column(String, nullable=False) aliases = Column(String, nullable=True) type = Column(String, nullable=False) diff --git a/hub/persistence/models/simulation_results.py b/hub/persistence/models/simulation_results.py index d4dea88d..a40c077e 100644 --- a/hub/persistence/models/simulation_results.py +++ b/hub/persistence/models/simulation_results.py @@ -19,8 +19,8 @@ class SimulationResults(Models): """ __tablename__ = 'simulation_results' id = Column(Integer, Sequence('simulation_results_id_seq'), primary_key=True) - city_id = Column(Integer, ForeignKey('city.id'), nullable=True) - city_object_id = Column(Integer, ForeignKey('city_object.id'), nullable=True) + city_id = Column(Integer, ForeignKey('city.id', ondelete='CASCADE'), nullable=True) + city_object_id = Column(Integer, ForeignKey('city_object.id', ondelete='CASCADE'), nullable=True) name = Column(String, nullable=False) values = Column(JSONB, nullable=False) created = Column(DateTime, default=datetime.datetime.utcnow) diff --git a/hub/persistence/repositories/application.py b/hub/persistence/repositories/application.py index f68fd0bb..a70fde37 100644 --- a/hub/persistence/repositories/application.py +++ b/hub/persistence/repositories/application.py @@ -10,6 +10,7 @@ import logging from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm.session import Session from hub.persistence.repository import Repository from hub.persistence.models import Application as Model @@ -48,10 +49,11 @@ class Application(Repository): pass try: application = Model(name=name, description=description, application_uuid=application_uuid) - self.session.add(application) - self.session.commit() - self.session.refresh(application) - return application.id + with Session(self.engine) as session: + session.add(application) + session.commit() + session.refresh(application) + return application.id except SQLAlchemyError as err: logging.error('An error occurred while creating application %s', err) raise SQLAlchemyError from err @@ -65,10 +67,11 @@ class Application(Repository): :return: None """ try: - self.session.query(Model).filter( - Model.application_uuid == application_uuid - ).update({'name': name, 'description': description, 'updated': datetime.datetime.utcnow()}) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter( + Model.application_uuid == application_uuid + ).update({'name': name, 'description': description, 'updated': datetime.datetime.utcnow()}) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating application %s', err) raise SQLAlchemyError from err @@ -80,9 +83,10 @@ class Application(Repository): :return: None """ try: - self.session.query(Model).filter(Model.application_uuid == application_uuid).delete() - self.session.flush() - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.application_uuid == application_uuid).delete() + session.flush() + session.commit() except SQLAlchemyError as err: logging.error('Error while deleting application %s', err) raise SQLAlchemyError from err @@ -94,10 +98,11 @@ class Application(Repository): :return: Application with the provided application_uuid """ try: - result_set = self.session.execute(select(Model).where( - Model.application_uuid == application_uuid) - ).first() - return result_set[0] + with Session(self.engine) as session: + result_set = session.execute(select(Model).where( + Model.application_uuid == application_uuid) + ).first() + return result_set[0] except SQLAlchemyError as err: logging.error('Error while fetching application by application_uuid %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repositories/city.py b/hub/persistence/repositories/city.py index 9c20346f..d1e07343 100644 --- a/hub/persistence/repositories/city.py +++ b/hub/persistence/repositories/city.py @@ -54,18 +54,18 @@ class City(Repository): application_id, user_id, __version__) - - self.session.add(db_city) - self.session.flush() - self.session.commit() - for building in city.buildings: - db_city_object = CityObject(db_city.id, - building) - self.session.add(db_city_object) - self.session.flush() - self.session.commit() - self.session.refresh(db_city) - return db_city.id + with Session(self.engine) as session: + session.add(db_city) + session.flush() + session.commit() + for building in city.buildings: + db_city_object = CityObject(db_city.id, + building) + session.add(db_city_object) + session.flush() + session.commit() + session.refresh(db_city) + return db_city.id except SQLAlchemyError as err: logging.error('An error occurred while creating a city %s', err) raise SQLAlchemyError from err @@ -79,8 +79,9 @@ class City(Repository): """ try: now = datetime.datetime.utcnow() - self.session.query(Model).filter(Model.id == city_id).update({'name': city.name, 'updated': now}) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.id == city_id).update({'name': city.name, 'updated': now}) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating city %s', err) raise SQLAlchemyError from err @@ -92,9 +93,10 @@ class City(Repository): :return: None """ try: - self.session.query(CityObject).filter(CityObject.city_id == city_id).delete() - self.session.query(Model).filter(Model.id == city_id).delete() - self.session.commit() + with Session(self.engine) as session: + session.query(CityObject).filter(CityObject.city_id == city_id).delete() + session.query(Model).filter(Model.id == city_id).delete() + session.commit() except SQLAlchemyError as err: logging.error('Error while fetching city %s', err) raise SQLAlchemyError from err @@ -108,13 +110,12 @@ class City(Repository): :return: [ModelCity] """ try: - result_set = self.session.execute(select(Model).where(Model.user_id == user_id, - Model.application_id == application_id, - Model.scenario == scenario - )).all() - self.session.close() - self.session = Session(self.engine) - return result_set + with Session(self.engine) as session: + result_set = session.execute(select(Model).where(Model.user_id == user_id, + Model.application_id == application_id, + Model.scenario == scenario + )).all() + return result_set except SQLAlchemyError as err: logging.error('Error while fetching city by name %s', err) raise SQLAlchemyError from err @@ -127,10 +128,11 @@ class City(Repository): :return: ModelCity """ try: - result_set = self.session.execute( - select(Model).where(Model.user_id == user_id, Model.application_id == application_id) - ) - return [r[0] for r in result_set] + with Session(self.engine) as session: + result_set = session.execute( + select(Model).where(Model.user_id == user_id, Model.application_id == application_id) + ) + return [r[0] for r in result_set] except SQLAlchemyError as err: logging.error('Error while fetching city by name %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repositories/city_object.py b/hub/persistence/repositories/city_object.py index aa69a0e3..30287411 100644 --- a/hub/persistence/repositories/city_object.py +++ b/hub/persistence/repositories/city_object.py @@ -46,10 +46,11 @@ class CityObject(Repository): try: city_object = Model(city_id=city_id, building=building) - self.session.add(city_object) - self.session.flush() - self.session.commit() - self.session.refresh(city_object) + with Session(self.engine) as session: + session.add(city_object) + session.flush() + session.commit() + session.refresh(city_object) return city_object.id except SQLAlchemyError as err: logging.error('An error occurred while creating city_object %s', err) @@ -68,17 +69,18 @@ class CityObject(Repository): for usage in internal_zone.usages: object_usage = f'{object_usage}{usage.name}_{usage.percentage} ' object_usage = object_usage.rstrip() - self.session.query(Model).filter(Model.name == building.name, Model.city_id == city_id).update( - {'name': building.name, - 'alias': building.alias, - 'object_type': building.type, - 'year_of_construction': building.year_of_construction, - 'function': building.function, - 'usage': object_usage, - 'volume': building.volume, - 'area': building.floor_area, - 'updated': datetime.datetime.utcnow()}) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.name == building.name, Model.city_id == city_id).update( + {'name': building.name, + 'alias': building.alias, + 'object_type': building.type, + 'year_of_construction': building.year_of_construction, + 'function': building.function, + 'usage': object_usage, + 'volume': building.volume, + 'area': building.floor_area, + 'updated': datetime.datetime.utcnow()}) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating city object %s', err) raise SQLAlchemyError from err @@ -91,8 +93,9 @@ class CityObject(Repository): :return: None """ try: - self.session.query(Model).filter(Model.city_id == city_id, Model.name == name).delete() - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.city_id == city_id, Model.name == name).delete() + session.commit() except SQLAlchemyError as err: logging.error('Error while deleting application %s', err) raise SQLAlchemyError from err @@ -106,15 +109,14 @@ class CityObject(Repository): """ try: # search by name first - city_object = self.session.execute(select(Model).where(Model.name == name, Model.city_id == city_id)).first() - if city_object is not None: - return city_object[0] - # name not found, so search by alias instead - city_objects = self.session.execute( - select(Model).where(Model.aliases.contains(name), Model.city_id == city_id) - ).all() - self.session.close() - self.session = Session(self.engine) + with Session(self.engine) as session: + city_object = session.execute(select(Model).where(Model.name == name, Model.city_id == city_id)).first() + if city_object is not None: + return city_object[0] + # name not found, so search by alias instead + city_objects = session.execute( + select(Model).where(Model.aliases.contains(name), Model.city_id == city_id) + ).all() for city_object in city_objects: aliases = city_object[0].aliases.replace('{', '').replace('}', '').split(',') for alias in aliases: diff --git a/hub/persistence/repositories/simulation_results.py b/hub/persistence/repositories/simulation_results.py index 0147eafc..3f6faa48 100644 --- a/hub/persistence/repositories/simulation_results.py +++ b/hub/persistence/repositories/simulation_results.py @@ -10,6 +10,7 @@ import logging from sqlalchemy import or_ from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session from hub.persistence.repository import Repository from hub.persistence.models import City @@ -52,11 +53,12 @@ class SimulationResults(Repository): values=values, city_id=city_id, city_object_id=city_object_id) - self.session.add(simulation_result) - self.session.flush() - self.session.commit() - self.session.refresh(simulation_result) - return simulation_result.id + with Session(self.engine) as session: + session.add(simulation_result) + session.flush() + session.commit() + session.refresh(simulation_result) + return simulation_result.id except SQLAlchemyError as err: logging.error('An error occurred while creating city_object %s', err) raise SQLAlchemyError from err @@ -71,22 +73,23 @@ class SimulationResults(Repository): :return: None """ try: - if city_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_id == city_id).update( + with Session(self.engine) as session: + if city_id is not None: + session.query(Model).filter(Model.name == name, Model.city_id == city_id).update( { 'values': values, 'updated': datetime.datetime.utcnow() }) - self.session.commit() - elif city_object_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).update( - { - 'values': values, - 'updated': datetime.datetime.utcnow() - }) - self.session.commit() - else: - raise NotImplementedError('Missing either city_id or city_object_id') + session.commit() + elif city_object_id is not None: + session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).update( + { + 'values': values, + 'updated': datetime.datetime.utcnow() + }) + session.commit() + else: + raise NotImplementedError('Missing either city_id or city_object_id') except SQLAlchemyError as err: logging.error('Error while updating city object %s', err) raise SQLAlchemyError from err @@ -100,14 +103,15 @@ class SimulationResults(Repository): :return: None """ try: - if city_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_id == city_id).delete() - self.session.commit() - elif city_object_id is not None: - self.session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).delete() - self.session.commit() - else: - raise NotImplementedError('Missing either city_id or city_object_id') + with Session(self.engine) as session: + if city_id is not None: + session.query(Model).filter(Model.name == name, Model.city_id == city_id).delete() + session.commit() + elif city_object_id is not None: + session.query(Model).filter(Model.name == name, Model.city_object_id == city_object_id).delete() + session.commit() + else: + raise NotImplementedError('Missing either city_id or city_object_id') except SQLAlchemyError as err: logging.error('Error while deleting application: %s', err) raise SQLAlchemyError from err @@ -119,7 +123,8 @@ class SimulationResults(Repository): :return: [City] with the provided city_id """ try: - return self.session.execute(select(City).where(City.id == city_id)).first() + with Session(self.engine) as session: + return session.execute(select(City).where(City.id == city_id)).first() except SQLAlchemyError as err: logging.error('Error while fetching city by city_id: %s', err) raise SQLAlchemyError from err @@ -131,7 +136,8 @@ class SimulationResults(Repository): :return: [CityObject] with the provided city_object_id """ try: - return self.session.execute(select(CityObject).where(CityObject.id == city_object_id)).first() + with Session(self.engine) as session: + return session.execute(select(CityObject).where(CityObject.id == city_object_id)).first() except SQLAlchemyError as err: logging.error('Error while fetching city by city_id: %s', err) raise SQLAlchemyError from err @@ -145,18 +151,19 @@ class SimulationResults(Repository): :return: [SimulationResult] """ try: - result_set = self.session.execute(select(Model).where(or_( - Model.city_id == city_id, - Model.city_object_id == city_object_id - ))) - results = [r[0] for r in result_set] - if not result_names: - return results - filtered_results = [] - for result in results: - if result.name in result_names: - filtered_results.append(result) - return filtered_results + with Session(self.engine) as session: + result_set = session.execute(select(Model).where(or_( + Model.city_id == city_id, + Model.city_object_id == city_object_id + ))) + results = [r[0] for r in result_set] + if not result_names: + return results + filtered_results = [] + for result in results: + if result.name in result_names: + filtered_results.append(result) + return filtered_results except SQLAlchemyError as err: logging.error('Error while fetching city by city_id: %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repositories/user.py b/hub/persistence/repositories/user.py index 51101858..4467f7cb 100644 --- a/hub/persistence/repositories/user.py +++ b/hub/persistence/repositories/user.py @@ -9,6 +9,7 @@ import logging from sqlalchemy import select from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import Session from hub.helpers.auth import Auth from hub.persistence.repository import Repository @@ -49,10 +50,11 @@ class User(Repository): pass try: user = Model(name=name, password=Auth.hash_password(password), role=role, application_id=application_id) - self.session.add(user) - self.session.flush() - self.session.commit() - self.session.refresh(user) + with Session(self.engine) as session: + session.add(user) + session.flush() + session.commit() + session.refresh(user) return user.id except SQLAlchemyError as err: logging.error('An error occurred while creating user %s', err) @@ -68,13 +70,14 @@ class User(Repository): :return: None """ try: - self.session.query(Model).filter(Model.id == user_id).update({ - 'name': name, - 'password': Auth.hash_password(password), - 'role': role, - 'updated': datetime.datetime.utcnow() - }) - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.id == user_id).update({ + 'name': name, + 'password': Auth.hash_password(password), + 'role': role, + 'updated': datetime.datetime.utcnow() + }) + session.commit() except SQLAlchemyError as err: logging.error('Error while updating user: %s', err) raise SQLAlchemyError from err @@ -86,8 +89,9 @@ class User(Repository): :return: None """ try: - self.session.query(Model).filter(Model.id == user_id).delete() - self.session.commit() + with Session(self.engine) as session: + session.query(Model).filter(Model.id == user_id).delete() + session.commit() except SQLAlchemyError as err: logging.error('Error while fetching user: %s', err) raise SQLAlchemyError from err @@ -100,10 +104,12 @@ class User(Repository): :return: User matching the search criteria or None """ try: - user = self.session.execute( - select(Model).where(Model.name == name, Model.application_id == application_id) - ).first() - return user[0] + with Session(self.engine) as session: + user = session.execute( + select(Model).where(Model.name == name, Model.application_id == application_id) + ).first() + session.commit() + return user[0] except SQLAlchemyError as err: logging.error('Error while fetching user by name and application: %s', err) raise SQLAlchemyError from err @@ -120,12 +126,13 @@ class User(Repository): :return: User """ try: - user = self.session.execute( - select(Model).where(Model.name == name, Model.application_id == application_id) - ).first() - if user: - if Auth.check_password(password, user[0].password): - return user[0] + with Session(self.engine) as session: + user = session.execute( + select(Model).where(Model.name == name, Model.application_id == application_id) + ).first() + if user: + if Auth.check_password(password, user[0].password): + return user[0] except SQLAlchemyError as err: logging.error('Error while fetching user by name: %s', err) raise SQLAlchemyError from err @@ -140,10 +147,11 @@ class User(Repository): :return: User """ try: - application = self.session.execute( - select(ApplicationModel).where(ApplicationModel.application_uuid == application_uuid) - ).first() - return self.get_by_name_application_id_and_password(name, password, application[0].id) + with Session(self.engine) as session: + application = session.execute( + select(ApplicationModel).where(ApplicationModel.application_uuid == application_uuid) + ).first() + return self.get_by_name_application_id_and_password(name, password, application[0].id) except SQLAlchemyError as err: logging.error('Error while fetching user by name: %s', err) raise SQLAlchemyError from err diff --git a/hub/persistence/repository.py b/hub/persistence/repository.py index 97ed9f91..5a3b4e26 100644 --- a/hub/persistence/repository.py +++ b/hub/persistence/repository.py @@ -6,7 +6,6 @@ Project Coder Peter Yefi peteryefi@gmail.com """ import logging from sqlalchemy import create_engine -from sqlalchemy.orm import Session from hub.persistence.configuration import Configuration @@ -19,6 +18,5 @@ class Repository: try: self.configuration = Configuration(db_name, dotenv_path, app_env) self.engine = create_engine(self.configuration.connection_string) - self.session = Session(self.engine) except ValueError as err: logging.error('Missing value for credentials: %s', err) diff --git a/tests/test_db_factory.py b/tests/test_db_factory.py index 94b599a7..4c216ad2 100644 --- a/tests/test_db_factory.py +++ b/tests/test_db_factory.py @@ -103,16 +103,16 @@ class Control: app_env='TEST', dotenv_path=dotenv_path) - self._application_uuid = '60b7fc1b-f389-4254-9ffd-22a4cf32c7a3' + self._application_uuid = 'b9e0ce80-1218-410c-8a64-9d9b7026aad8' self._application_id = 1 self._user_id = 1 self._application_id = self._database.persist_application( - 'City_layers', - 'City layers test user', + 'test', + 'test', self.application_uuid ) - self._user_id = self._database.create_user('city_layers', self._application_id, 'city_layers', UserRoles.Admin) + self._user_id = self._database.create_user('test', self._application_id, 'test', UserRoles.Admin) self._pickle_path = Path('tests_data/pickle_path.bz2').resolve() From f865490ff908d239c46b3ae8c544be6ff39cc63c Mon Sep 17 00:00:00 2001 From: guille Date: Fri, 13 Oct 2023 09:11:03 +0200 Subject: [PATCH 7/8] Corrected bug in distribution_systems_electrical_consumption, but need to be confirmed with pilar --- hub/city_model_structure/building.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hub/city_model_structure/building.py b/hub/city_model_structure/building.py index d7efb9f2..e4d654b2 100644 --- a/hub/city_model_structure/building.py +++ b/hub/city_model_structure/building.py @@ -743,8 +743,11 @@ class Building(CityObject): for key, item in self._distribution_systems_electrical_consumption.items(): for i in range(0, len(item)): + _working_hours_value = _working_hours[key] + if len(item) == 12: + _working_hours_value = _working_hours[key][i] self._distribution_systems_electrical_consumption[key][i] += ( - _peak_load * _consumption_fix_flow * _working_hours[key] * cte.WATTS_HOUR_TO_JULES + _peak_load * _consumption_fix_flow * _working_hours_value * cte.WATTS_HOUR_TO_JULES ) return self._distribution_systems_electrical_consumption From 2afce4acce828e389246e2fa4dda88afef8de191 Mon Sep 17 00:00:00 2001 From: guille Date: Thu, 19 Oct 2023 07:39:35 +0200 Subject: [PATCH 8/8] Corrected obj and glb exporters, building now exposes lower and upper corner --- hub/city_model_structure/building.py | 14 ++++++++++++++ hub/exports/formats/glb.py | 11 ++--------- hub/exports/formats/obj.py | 13 +++++-------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/hub/city_model_structure/building.py b/hub/city_model_structure/building.py index e4d654b2..5a6e109e 100644 --- a/hub/city_model_structure/building.py +++ b/hub/city_model_structure/building.py @@ -810,3 +810,17 @@ class Building(CityObject): orientation_losses_factor[_key]['south'])] self._onsite_electrical_production[_key] = _results return self._onsite_electrical_production + + @property + def lower_corner(self): + """ + Get building lower corner. + """ + return [self._min_x, self._min_y, self._min_z] + + @property + def upper_corner(self): + """ + Get building upper corner. + """ + return [self._max_x, self._max_y, self._max_z] diff --git a/hub/exports/formats/glb.py b/hub/exports/formats/glb.py index 67b5a960..6329788d 100644 --- a/hub/exports/formats/glb.py +++ b/hub/exports/formats/glb.py @@ -1,4 +1,3 @@ -import glob import os import shutil import subprocess @@ -33,24 +32,18 @@ class Glb: def _export(self): try: for building in self._city.buildings: - print(self._city.lower_corner, self._city.upper_corner) city = City(self._city.lower_corner, self._city.upper_corner, self._city.srs_name) - city.name = building.name city.add_city_object(building) + city.name = building.name Obj(city, self._path) glb = f'{self._path}/{building.name}.glb' subprocess.run([ self._obj2gtl, '-i', f'{self._path}/{building.name}.obj', '-b', - '-o', f'{glb}', - '--triangleWindingOrderSanitization', - '--inputUpAxis', 'Y', - '--outputUpAxis', 'Y' + '-o', f'{glb}' ]) - os.unlink(f'{self._path}/{building.name}.obj') os.unlink(f'{self._path}/{building.name}.mtl') except (subprocess.SubprocessError, subprocess.TimeoutExpired, subprocess.CalledProcessError) as err: raise GltExceptionError from err - diff --git a/hub/exports/formats/obj.py b/hub/exports/formats/obj.py index 219bb1ac..d212740c 100644 --- a/hub/exports/formats/obj.py +++ b/hub/exports/formats/obj.py @@ -4,7 +4,6 @@ SPDX - License - Identifier: LGPL - 3.0 - or -later Copyright © 2022 Concordia CERC group Project Coder Guille Gutierrez guillermo.gutierrezmorote@concordia.ca """ -import math from pathlib import Path import numpy as np @@ -27,7 +26,7 @@ class Obj: def _to_vertex(self, coordinate): x, y, z = self._ground(coordinate) - return f'v {x} {z} {y}\n' + return f'v {x} {z} -{y}\n' # to match opengl expectations def _to_texture_vertex(self, coordinate): u, v, _ = self._ground(coordinate) @@ -55,22 +54,22 @@ class Obj: with open(mtl_file_path, 'w', encoding='utf-8') as mtl: mtl.write("newmtl cerc_base_material\n") mtl.write("Ka 1.0 1.0 1.0 # Ambient color (white)\n") - mtl.write("Kd 0.3 0.1 0.3 # Diffuse color (greenish)\n") + mtl.write("Kd 0.1 0.3 0.1 # Diffuse color (greenish)\n") mtl.write("Ks 1.0 1.0 1.0 # Specular color (white)\n") mtl.write("Ns 400.0 # Specular exponent (defines shininess)\n") vertices = {} - normals_index = {} faces = [] vertex_index = 0 normal_index = 0 with open(obj_file_path, 'w', encoding='utf-8') as obj: obj.write("# cerc-hub export\n") - obj.write(f'mtllib {mtl_name}') + obj.write(f'mtllib {mtl_name}\n') for building in self._city.buildings: obj.write(f'# building {building.name}\n') obj.write(f'g {building.name}\n') obj.write('s off\n') + for surface in building.surfaces: obj.write(f'# surface {surface.name}\n') face = [] @@ -79,7 +78,6 @@ class Obj: textures = [] for coordinate in surface.perimeter_polygon.coordinates: vertex = self._to_vertex(coordinate) - if vertex not in vertices: vertex_index += 1 vertices[vertex] = vertex_index @@ -88,8 +86,7 @@ class Obj: textures.append(self._to_texture_vertex(coordinate)) # only append if non-existing else: current = vertices[vertex] - - face.insert(0, f'{current}/{current}/{normal_index}') # insert counterclockwise + face.append(f'{current}/{current}/{normal_index}') # insert clockwise obj.writelines(normal) # add the normal obj.writelines(textures) # add the texture vertex