s_ranjbar/city_retrofit
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug fix and generalization for more concordia sensors

Browse Source
This commit is contained in:
Guille Gutierrez 2021-06-02 11:16:00 -04:00
parent 2f0da19293
commit 4d821880a4
13 changed files with 243 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
data/sensors/concordia.json
View File

@ -1,11 +0,0 @@
{
"sensors": [
{ "building" : "EV",
"sensors": ["COMPTEUR.SQD.017.IC:POWER 3P", "COMPTEUR.SQD.B1.IC:POWER 3P", "COMPTEUR.SQD.B2.IC:POWER 3P",
"TOTKWEV-MB.IC"]
},
{ "building" : "GM",
"sensors": ["MDICOR.GM"]
}
]
}

8
data/sensors/concordia_energy_db.json Normal file
View File

@ -0,0 +1,8 @@
{
"sensors": [
{ "building" : "EV",
"sensors": ["TOTKWCH3.IC","TOTKWEV.IC","COMPTEUR.SQD.017.IC:POWER 3P", "COMPTEUR.SQD.B1.IC:POWER 3P", "COMPTEUR.SQD.B2.IC:POWER 3P",
"TOTKWEV-MB.IC"]
}
]
}

21
helpers/geometry_helper.py
View File

@ -47,16 +47,6 @@ class GeometryHelper:
delta = math.fabs(a1 - a2)
return delta <= self._area_delta
def almost_equal(self, delta_max, v1, v2):
"""
Compare two points and decides if they are almost equal (distance under delta_max)
:param delta_max: maximum distance to be considered same point
:param v1: [x,y,z]
:param v2: [x,y,z]
:return: Boolean
"""
delta = self.distance_between_points(v1, v2)
return delta <= delta_max
def is_almost_same_surface(self, s1, s2):
"""
@ -95,17 +85,6 @@ class GeometryHelper:
else:
return True
@staticmethod
def to_points_matrix(points):
"""
Transform a point vector into a point matrix
:param points: [x, y, z, x, y, z ...]
:return: [[x,y,z],[x,y,z]...]
"""
rows = points.size // 3
points = points.reshape(rows, 3)
return points
@staticmethod
def segment_list_to_trimesh(lines) -> Trimesh:
line_points = [lines[0][0], lines[0][1]]

102
imports/geometry/citygml.py
View File

@ -8,9 +8,10 @@ import xmltodict
from city_model_structure.city import City
from city_model_structure.building import Building
from city_model_structure.attributes.surface import Surface
from helpers.geometry_helper import GeometryHelper
from city_model_structure.attributes.polygon import Polygon
from imports.geometry.citygml_lod2 import CityGmlLod2
from imports.geometry.citygml_lod1 import CityGmlLod1
class CityGml:
@ -73,27 +74,29 @@ class CityGml:
# todo: refactor this method to clearly choose the gml type
self._city = City(self._lower_corner, self._upper_corner, self._srs_name)
i = 0
building_part = None
for o in self._gml['CityModel']['cityObjectMember']:
i += 1
lod = 0
surfaces = []
if 'lod1Solid' in o['Building']:
lod += 1
surfaces = CityGml._lod1_solid(o)
surfaces = CityGmlLod1.lod1_solid(o)
elif 'lod1MultiSurface' in o['Building']:
lod += 1
surfaces = CityGml._lod1_multi_surface(o)
surfaces = CityGmlLod1.lod1_multi_surface(o)
elif 'lod2Solid' in o['Building']:
lod += 1
surfaces = CityGmlLod2.lod2_solid(o)
elif 'lod2MultiSurface' in o['Building']:
# todo: check if this is a real case or a miss-formed citygml
lod = 2
surfaces = surfaces + CityGml._lod2_solid_multi_surface(o)
surfaces = surfaces + CityGmlLod2.lod2_solid_multi_surface(o)
else:
for bound in o['Building']['boundedBy']:
surface_type = next(iter(bound))
if 'lod2MultiSurface' in bound[surface_type]:
lod = 2
surfaces = surfaces + CityGml._lod2(bound)
surfaces = surfaces + CityGmlLod2.lod2(bound)
if 'lod3Solid' in o['Building']:
lod += 4
if 'lod4Solid' in o['Building']:
@ -106,15 +109,6 @@ class CityGml:
function = None
year_of_construction = None
if 'consistsOfBuildingPart' in o['Building']:
if 'BuildingPart' in o['Building']['consistsOfBuildingPart']:
name = o['Building']['consistsOfBuildingPart']['BuildingPart']['name']
if 'yearOfConstruction' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
year_of_construction = o['Building']['consistsOfBuildingPart']['BuildingPart']['yearOfConstruction']
if 'function' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
function = o['Building']['consistsOfBuildingPart']['BuildingPart']['function']
else:
name = o['Building']['@id']
if 'yearOfConstruction' in o['Building']:
year_of_construction = o['Building']['yearOfConstruction']
@ -138,84 +132,6 @@ class CityGml:
terrains.append(curve_points)
return terrains
@staticmethod
def _lod1_solid(o):
try:
solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']['#text']))
for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
except TypeError:
solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]
@staticmethod
def _lod1_multi_surface(o):
solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['lod1MultiSurface']['MultiSurface']['surfaceMember']]
return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]
@staticmethod
def _lod2_solid_multi_surface(o):
if 'boundedBy' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
if 'RoofSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']['lod2MultiSurface'] != 'None':
polygons = [Polygon(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
elif 'WallSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface'] != 'None':
polygons = [Polygon(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
else:
polygons = [Polygon(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
return [Surface(p,p) for p in polygons]
@staticmethod
def _lod2_composite_surface(s):
solid_points = [CityGml._solid_points((CityGml._remove_last_point(sm['Polygon']['exterior']['LinearRing']['posList'])))
for sm in s['CompositeSurface']['surfaceMember']]
return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]
@staticmethod
def _lod2_multi_surface(s, surface_type):
# todo: this need to be changed into surface bounded?
try:
solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']
['#text']))]
except TypeError:
solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']
['posList']))]
return [Surface(Polygon(sp),Polygon(sp), surface_type=surface_type) for sp in solid_points]
@staticmethod
def _lod2(bound):
surfaces = []
for surface_type in iter(bound):
for s in bound[surface_type]['lod2MultiSurface']['MultiSurface']['surfaceMember']:
if 'CompositeSurface' in s:
surfaces = surfaces + CityGml._lod2_composite_surface(s)
else:
surfaces = surfaces + CityGml._lod2_multi_surface(s, surface_type)
return surfaces
@staticmethod
def _remove_last_point(points):
array = points.split(' ')
res = " "
return res.join(array[0:len(array) - 3])
@staticmethod
def _solid_points(coordinates) -> np.ndarray:
"""
Solid surface point matrix [[x, y, z],[x, y, z],...]
:parameter coordinates: string from file
:return: np.ndarray
"""
solid_points = np.fromstring(coordinates, dtype=float, sep=' ')
solid_points = GeometryHelper.to_points_matrix(solid_points)
return solid_points
@staticmethod
def _holes_points(holes_coordinates) -> [np.ndarray]:

25
imports/geometry/citygml_lod1.py Normal file
View File

@ -0,0 +1,25 @@
from imports.geometry.citygml_tools import CityGmlTools
from city_model_structure.attributes.surface import Surface
from city_model_structure.attributes.polygon import Polygon
class CityGmlLod1(CityGmlTools):
@staticmethod
def lod1_solid(o):
try:
solid_points = [
CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']['#text']))
for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
except TypeError:
solid_points = [
CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
return [Surface(Polygon(sp), Polygon(sp)) for sp in solid_points]
@staticmethod
def lod1_multi_surface(o):
solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['lod1MultiSurface']['MultiSurface']['surfaceMember']]
return [Surface(Polygon(sp), Polygon(sp)) for sp in solid_points]

65
imports/geometry/citygml_lod2.py Normal file
View File

@ -0,0 +1,65 @@
from imports.geometry.citygml_tools import CityGmlTools
from city_model_structure.attributes.surface import Surface
from city_model_structure.attributes.polygon import Polygon
class CityGmlLod2(CityGmlTools):
@staticmethod
def _lod2_composite_surface(s):
solid_points = [
CityGmlTools._solid_points((CityGmlTools._remove_last_point(sm['Polygon']['exterior']['LinearRing']['posList'])))
for sm in s['CompositeSurface']['surfaceMember']]
return [Surface(Polygon(sp), Polygon(sp)) for sp in solid_points]
@staticmethod
def _lod2_multi_surface(s, surface_type):
# todo: this need to be changed into surface bounded?
try:
solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(
s['Polygon']['exterior']['LinearRing']['posList']['#text']))]
except TypeError:
solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']
['posList']))]
return [Surface(Polygon(sp), Polygon(sp), surface_type=surface_type) for sp in solid_points]
@staticmethod
def lod2(bound):
surfaces = []
for surface_type in iter(bound):
for s in bound[surface_type]['lod2MultiSurface']['MultiSurface']['surfaceMember']:
if 'CompositeSurface' in s:
surfaces = surfaces + CityGmlLod2._lod2_composite_surface(s)
else:
surfaces = surfaces + CityGmlLod2._lod2_multi_surface(s, surface_type)
return surfaces
@staticmethod
def lod2_solid_multi_surface(o):
polygons = None
if 'boundedBy' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
if 'RoofSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']['lod2MultiSurface'] != 'None':
polygons = [Polygon(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']
['lod2MultiSurface']['MultiSurface']['surfaceMember']]
elif 'WallSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface'] != 'None':
polygons = [Polygon(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
else:
polygons = [Polygon(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
return [Surface(p,p) for p in polygons]
@staticmethod
def lod2_solid(o):
try:
solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']['#text']))
for s in o['Building']['lod2Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
except TypeError:
solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
for s in o['Building']['lod2Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]

22
imports/geometry/citygml_tools.py Normal file
View File

@ -0,0 +1,22 @@
import numpy as np
from imports.geometry.helpers.geometry_helper import GeometryHelper
class CityGmlTools:
@staticmethod
def _remove_last_point(points):
array = points.split(' ')
res = " "
return res.join(array[0:len(array) - 3])
@staticmethod
def _solid_points(coordinates) -> np.ndarray:
"""
Solid surface point matrix [[x, y, z],[x, y, z],...]
:parameter coordinates: string from file
:return: np.ndarray
"""
solid_points = np.fromstring(coordinates, dtype=float, sep=' ')
solid_points = GeometryHelper.to_points_matrix(solid_points)
return solid_points

22
imports/geometry/helpers/geometry_helper.py
View File

@ -293,3 +293,25 @@ class GeometryHelper:
:return: str
"""
return GeometryHelper.fuction_to_usage[building_function]
@staticmethod
def to_points_matrix(points):
"""
Transform a point vector into a point matrix
:param points: [x, y, z, x, y, z ...]
:return: [[x,y,z],[x,y,z]...]
"""
rows = points.size // 3
points = points.reshape(rows, 3)
return points
def almost_equal(self, delta_max, v1, v2):
"""
Compare two points and decides if they are almost equal (distance under delta_max)
:param delta_max: maximum distance to be considered same point
:param v1: [x,y,z]
:param v2: [x,y,z]
:return: Boolean
"""
delta = self.distance_between_points(v1, v2)
return delta <= delta_max

74
imports/sensors/concordia_energy_consumption.py
View File

@ -3,61 +3,19 @@ Concordia energy consumption
SPDX - License - Identifier: LGPL - 3.0 - or -later
Copyright © 2021 Project Author Guille Gutierrez guillermo.gutierrezmorote@concordia.ca
"""
import io
import json
from pathlib import Path
import pandas as pd
from imports.sensors.concordia_file_report import ConcordiaFileReport
from city_model_structure.attributes.concordia_energy_sensor import ConcordiaEnergySensor
class ConcordiaEnergyConsumption:
class ConcordiaEnergyConsumption(ConcordiaFileReport):
def __init__(self, city, end_point, base_path):
self._buildings = []
self._sensors = []
self._sensor_point = {}
self._city = city
self._end_point = end_point
self._sensor_database = base_path
metadata = True
content = False
with open(Path(base_path / 'concordia.json').resolve()) as concordia_db:
self._sensor_database = json.load(concordia_db)
for building in self._sensor_database['sensors']:
building_name = building['building']
for sensor in building['sensors']:
self._buildings.append(building_name)
self._sensors.append(sensor)
buffer = ""
with open(end_point.resolve()) as data:
for line in data:
line = ConcordiaEnergyConsumption.clean_line(line)
if metadata:
fields = line.split(',')
if len(fields) > 2:
point = fields[0].replace(":", "")
key = fields[1]
if fields[1] in self._sensors:
self._sensor_point[key] = point
if "End of Report" in line:
content = False
if content:
line = ConcordiaEnergyConsumption.merge_date_time(line)
buffer = buffer + line + '\n'
if line is '':
metadata = False
content = True
measures = pd.read_csv(io.StringIO(buffer), sep=',')
measures["Date time"] = pd.to_datetime(measures["Date time"])
measures = ConcordiaEnergyConsumption.force_format(measures)
super().__init__(city, end_point, base_path, 'concordia_energy_db.json')
for building in city.buildings:
for i in range(len(self._buildings)):
if self._buildings[i] == building.name:
building_measures = [measures["Date time"], measures[self._sensor_point[self._sensors[i]]]]
if self._buildings[i] == building.name and self._sensors[i] in self._sensor_point:
building_measures = [self._measures["Date time"], self._measures[self._sensor_point[self._sensors[i]]]]
building_headers = ["Date time", "Energy consumption"]
building_energy_consumption = pd.concat(building_measures, keys=building_headers, axis=1)
sensor = ConcordiaEnergySensor(self._sensors[i])
@ -71,25 +29,3 @@ class ConcordiaEnergyConsumption:
sensor.add_period(building_energy_consumption)
building.sensors.append(sensor)
@staticmethod
def clean_line(line):
return line.replace('"', '').replace('\n', '')
@staticmethod
def merge_date_time(line):
fields = line.split(',')
date = fields[0]
time = fields[1]
if '<>' in date:
return line.replace(f'{date},{time}', 'Date time')
else:
date_fields = date.split('/')
format_date_time = f'"{int(date_fields[2])}-{int(date_fields[0]):02d}-{int(date_fields[1]):02d} {time}"'
return line.replace(f'{date},{time}', format_date_time)
@staticmethod
def force_format(df):
for head in df.head():
if 'Date time' not in head:
df = df.astype({head: 'float64'})
return df

77
imports/sensors/concordia_file_report.py Normal file
View File

@ -0,0 +1,77 @@
"""
Concordia file report
SPDX - License - Identifier: LGPL - 3.0 - or -later
Copyright © 2021 Project Author Guille Gutierrez guillermo.gutierrezmorote@concordia.ca
"""
import io
import json
from pathlib import Path
import pandas as pd
from city_model_structure.attributes.concordia_energy_sensor import ConcordiaEnergySensor
class ConcordiaFileReport:
def __init__(self, city, end_point, base_path, db_file):
self._buildings = []
self._sensors = []
self._sensor_point = {}
self._city = city
self._end_point = end_point
self._sensor_database = base_path
metadata = True
content = False
with open(Path(base_path / db_file).resolve()) as concordia_db:
self._sensor_database = json.load(concordia_db)
for building in self._sensor_database['sensors']:
building_name = building['building']
for sensor in building['sensors']:
self._buildings.append(building_name)
self._sensors.append(sensor)
buffer = ""
with open(end_point.resolve()) as data:
for line in data:
line = ConcordiaFileReport.clean_line(line)
if metadata:
fields = line.split(',')
if len(fields) > 2:
point = fields[0].replace(":", "")
key = fields[1]
if fields[1] in self._sensors:
self._sensor_point[key] = point
if "End of Report" in line:
content = False
if content:
line = ConcordiaFileReport.merge_date_time(line)
buffer = buffer + line + '\n'
if line is '':
metadata = False
content = True
measures = pd.read_csv(io.StringIO(buffer), sep=',')
measures["Date time"] = pd.to_datetime(measures["Date time"])
self._measures = ConcordiaFileReport.force_format(measures)
@staticmethod
def clean_line(line):
return line.replace('"', '').replace('\n', '')
@staticmethod
def merge_date_time(line):
fields = line.split(',')
date = fields[0]
time = fields[1]
if '<>' in date:
return line.replace(f'{date},{time}', 'Date time')
else:
date_fields = date.split('/')
format_date_time = f'"{int(date_fields[2])}-{int(date_fields[0]):02d}-{int(date_fields[1]):02d} {time}"'
return line.replace(f'{date},{time}', format_date_time)
@staticmethod
def force_format(df):
for head in df.head():
if 'Date time' not in head:
df = df.astype({head: 'float64'})
return df

3
imports/sensors_factory.py
View File

@ -20,6 +20,9 @@ class SensorsFactory:
def _cec(self):
ConcordiaEnergyConsumption(self._city, self._end_point, self._base_path)
def _cgf(self):
ConcordiaGasFlow(self._city, self._end_point, self._base_path)
def enrich(self):
"""
Enrich the city with the usages information

2
non_functional_tests/test_geometry_factory.py
View File

@ -29,7 +29,7 @@ class TestGeometryFactory(TestCase):
return self._city
def _get_obj(self, file):
# todo: solve the incongruences between city and city_debug
# todo: solve the incongruities between city and city_debug
file_path = (self._example_path / file).resolve()
self._city = GeometryFactory('obj', file_path)._city_debug
self.assertIsNotNone(self._city, 'city is none')

1
non_functional_tests/test_sensors_factory.py
View File

@ -49,3 +49,4 @@ class TestSensorsFactory(TestCase):
sensor.add_period(update)
row = sensor.measures.loc[sensor.measures["Date time"] == '2020-01-19 23:55:00']['Energy consumption'].iloc[0]
self.assertTrue(f'{row}' == '12345.0')