Bug fix and generalization for more concordia sensors

data/sensors/concordia.json

@@ -1,11 +0,0 @@
-{
-  "sensors": [
-    { "building" :  "EV",
-      "sensors": ["COMPTEUR.SQD.017.IC:POWER 3P", "COMPTEUR.SQD.B1.IC:POWER 3P", "COMPTEUR.SQD.B2.IC:POWER 3P",
-        "TOTKWEV-MB.IC"]
-    },
-    { "building" :  "GM",
-      "sensors": ["MDICOR.GM"]
-    }
-  ]
-}

data/sensors/concordia_energy_db.json

@@ -0,0 +1,8 @@
+{
+  "sensors": [
+    { "building" :  "EV",
+      "sensors": ["TOTKWCH3.IC","TOTKWEV.IC","COMPTEUR.SQD.017.IC:POWER 3P", "COMPTEUR.SQD.B1.IC:POWER 3P", "COMPTEUR.SQD.B2.IC:POWER 3P",
+        "TOTKWEV-MB.IC"]
+    }
+  ]
+}

helpers/geometry_helper.py

@@ -47,16 +47,6 @@ class GeometryHelper:
     delta = math.fabs(a1 - a2)
     return delta <= self._area_delta
 
-  def almost_equal(self, delta_max, v1, v2):
-    """
-    Compare two points and decides if they are almost equal (distance under delta_max)
-    :param delta_max: maximum distance to be considered same point
-    :param v1: [x,y,z]
-    :param v2: [x,y,z]
-    :return: Boolean
-    """
-    delta = self.distance_between_points(v1, v2)
-    return delta <= delta_max
 
   def is_almost_same_surface(self, s1, s2):
     """
@@ -95,17 +85,6 @@ class GeometryHelper:
     else:
       return True
 
-  @staticmethod
-  def to_points_matrix(points):
-    """
-    Transform a point vector into a point matrix
-    :param points: [x, y, z, x, y, z ...]
-    :return: [[x,y,z],[x,y,z]...]
-    """
-    rows = points.size // 3
-    points = points.reshape(rows, 3)
-    return points
-
   @staticmethod
   def segment_list_to_trimesh(lines) -> Trimesh:
     line_points = [lines[0][0], lines[0][1]]

imports/geometry/citygml.py

@@ -8,9 +8,10 @@ import xmltodict
 
 from city_model_structure.city import City
 from city_model_structure.building import Building
-from city_model_structure.attributes.surface import Surface
 from helpers.geometry_helper import GeometryHelper
 from city_model_structure.attributes.polygon import Polygon
+from imports.geometry.citygml_lod2 import CityGmlLod2
+from imports.geometry.citygml_lod1 import CityGmlLod1
 
 
 class CityGml:
@@ -73,27 +74,29 @@ class CityGml:
       # todo: refactor this method to clearly choose the gml type
       self._city = City(self._lower_corner, self._upper_corner, self._srs_name)
       i = 0
-      building_part = None
       for o in self._gml['CityModel']['cityObjectMember']:
         i += 1
         lod = 0
         surfaces = []
         if 'lod1Solid' in o['Building']:
           lod += 1
-          surfaces = CityGml._lod1_solid(o)
+          surfaces = CityGmlLod1.lod1_solid(o)
         elif 'lod1MultiSurface' in o['Building']:
           lod += 1
-          surfaces = CityGml._lod1_multi_surface(o)
+          surfaces = CityGmlLod1.lod1_multi_surface(o)
+        elif 'lod2Solid' in o['Building']:
+          lod += 1
+          surfaces = CityGmlLod2.lod2_solid(o)
         elif 'lod2MultiSurface' in o['Building']:
           # todo: check if this is a real case or a miss-formed citygml
           lod = 2
-          surfaces = surfaces + CityGml._lod2_solid_multi_surface(o)
+          surfaces = surfaces + CityGmlLod2.lod2_solid_multi_surface(o)
         else:
           for bound in o['Building']['boundedBy']:
             surface_type = next(iter(bound))
             if 'lod2MultiSurface' in bound[surface_type]:
               lod = 2
-              surfaces = surfaces + CityGml._lod2(bound)
+              surfaces = surfaces + CityGmlLod2.lod2(bound)
             if 'lod3Solid' in o['Building']:
               lod += 4
             if 'lod4Solid' in o['Building']:
@@ -106,20 +109,11 @@ class CityGml:
 
         function = None
         year_of_construction = None
-        if 'consistsOfBuildingPart' in o['Building']:
+        name = o['Building']['@id']
-          if 'BuildingPart' in o['Building']['consistsOfBuildingPart']:
+        if 'yearOfConstruction' in o['Building']:
-            name = o['Building']['consistsOfBuildingPart']['BuildingPart']['name']
+          year_of_construction = o['Building']['yearOfConstruction']
-            if 'yearOfConstruction' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
+        if 'function' in o['Building']:
-              year_of_construction = o['Building']['consistsOfBuildingPart']['BuildingPart']['yearOfConstruction']
+          function = o['Building']['function']
-            if 'function' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
-              function = o['Building']['consistsOfBuildingPart']['BuildingPart']['function']
-
-        else:
-          name = o['Building']['@id']
-          if 'yearOfConstruction' in o['Building']:
-            year_of_construction = o['Building']['yearOfConstruction']
-          if 'function' in o['Building']:
-            function = o['Building']['function']
         self._city.add_city_object(Building(name, lod, surfaces, year_of_construction, function, self._lower_corner,
                                             terrains))
     return self._city
@@ -138,84 +132,6 @@ class CityGml:
       terrains.append(curve_points)
     return terrains
 
-  @staticmethod
-  def _lod1_solid(o):
-    try:
-      solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']['#text']))
-                  for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
-    except TypeError:
-      solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
-                  for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
-
-    return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]
-
-  @staticmethod
-  def _lod1_multi_surface(o):
-    solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
-                for s in o['Building']['lod1MultiSurface']['MultiSurface']['surfaceMember']]
-    return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]
-
-  @staticmethod
-  def _lod2_solid_multi_surface(o):
-    if 'boundedBy' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
-      if 'RoofSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
-        if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']['lod2MultiSurface'] != 'None':
-          polygons = [Polygon(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
-                    for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
-
-      elif 'WallSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
-        if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface'] != 'None':
-          polygons = [Polygon(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
-                    for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
-    else:
-      polygons = [Polygon(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
-                  for s in o['Building']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
-    return [Surface(p,p) for p in polygons]
-
-  @staticmethod
-  def _lod2_composite_surface(s):
-    solid_points = [CityGml._solid_points((CityGml._remove_last_point(sm['Polygon']['exterior']['LinearRing']['posList'])))
-                for sm in s['CompositeSurface']['surfaceMember']]
-    return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]
-
-  @staticmethod
-  def _lod2_multi_surface(s, surface_type):
-    # todo: this need to be changed into surface bounded?
-    try:
-      solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']
-                                                                       ['#text']))]
-    except TypeError:
-      solid_points = [CityGml._solid_points(CityGml._remove_last_point(s['Polygon']['exterior']['LinearRing']
-                                                                         ['posList']))]
-    return [Surface(Polygon(sp),Polygon(sp), surface_type=surface_type) for sp in solid_points]
-
-  @staticmethod
-  def _lod2(bound):
-    surfaces = []
-    for surface_type in iter(bound):
-      for s in bound[surface_type]['lod2MultiSurface']['MultiSurface']['surfaceMember']:
-        if 'CompositeSurface' in s:
-          surfaces = surfaces + CityGml._lod2_composite_surface(s)
-        else:
-          surfaces = surfaces + CityGml._lod2_multi_surface(s, surface_type)
-    return surfaces
-
-  @staticmethod
-  def _remove_last_point(points):
-    array = points.split(' ')
-    res = " "
-    return res.join(array[0:len(array) - 3])
-
-  @staticmethod
-  def _solid_points(coordinates) -> np.ndarray:
-    """
-    Solid surface point matrix [[x, y, z],[x, y, z],...]
-    :parameter coordinates: string from file
-    :return: np.ndarray
-    """
-    solid_points = np.fromstring(coordinates, dtype=float, sep=' ')
-    solid_points = GeometryHelper.to_points_matrix(solid_points)
-    return solid_points
 
   @staticmethod
   def _holes_points(holes_coordinates) -> [np.ndarray]:

imports/geometry/citygml_lod1.py

@@ -0,0 +1,25 @@
+from imports.geometry.citygml_tools import CityGmlTools
+from city_model_structure.attributes.surface import Surface
+from city_model_structure.attributes.polygon import Polygon
+
+
+class CityGmlLod1(CityGmlTools):
+
+  @staticmethod
+  def lod1_solid(o):
+    try:
+      solid_points = [
+        CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']['#text']))
+        for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
+    except TypeError:
+      solid_points = [
+        CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
+        for s in o['Building']['lod1Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
+
+    return [Surface(Polygon(sp), Polygon(sp)) for sp in solid_points]
+
+  @staticmethod
+  def lod1_multi_surface(o):
+    solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
+                    for s in o['Building']['lod1MultiSurface']['MultiSurface']['surfaceMember']]
+    return [Surface(Polygon(sp), Polygon(sp)) for sp in solid_points]

imports/geometry/citygml_lod2.py

@@ -0,0 +1,65 @@
+from imports.geometry.citygml_tools import CityGmlTools
+from city_model_structure.attributes.surface import Surface
+from city_model_structure.attributes.polygon import Polygon
+
+
+class CityGmlLod2(CityGmlTools):
+
+  @staticmethod
+  def _lod2_composite_surface(s):
+    solid_points = [
+      CityGmlTools._solid_points((CityGmlTools._remove_last_point(sm['Polygon']['exterior']['LinearRing']['posList'])))
+      for sm in s['CompositeSurface']['surfaceMember']]
+    return [Surface(Polygon(sp), Polygon(sp)) for sp in solid_points]
+
+  @staticmethod
+  def _lod2_multi_surface(s, surface_type):
+    # todo: this need to be changed into surface bounded?
+    try:
+      solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(
+        s['Polygon']['exterior']['LinearRing']['posList']['#text']))]
+    except TypeError:
+      solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']
+                                                                       ['posList']))]
+    return [Surface(Polygon(sp), Polygon(sp), surface_type=surface_type) for sp in solid_points]
+
+  @staticmethod
+  def lod2(bound):
+    surfaces = []
+    for surface_type in iter(bound):
+      for s in bound[surface_type]['lod2MultiSurface']['MultiSurface']['surfaceMember']:
+        if 'CompositeSurface' in s:
+          surfaces = surfaces + CityGmlLod2._lod2_composite_surface(s)
+        else:
+          surfaces = surfaces + CityGmlLod2._lod2_multi_surface(s, surface_type)
+    return surfaces
+
+  @staticmethod
+  def lod2_solid_multi_surface(o):
+    polygons = None
+    if 'boundedBy' in o['Building']['consistsOfBuildingPart']['BuildingPart']:
+      if 'RoofSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
+        if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']['lod2MultiSurface'] != 'None':
+          polygons = [Polygon(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
+                    for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['RoofSurface']
+                      ['lod2MultiSurface']['MultiSurface']['surfaceMember']]
+
+      elif 'WallSurface' in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']:
+        if o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface'] != 'None':
+          polygons = [Polygon(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
+                    for s in o['Building']['consistsOfBuildingPart']['BuildingPart']['boundedBy']['WallSurface']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
+    else:
+      polygons = [Polygon(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
+                  for s in o['Building']['lod2MultiSurface']['MultiSurface']['surfaceMember']]
+    return [Surface(p,p) for p in polygons]
+
+
+  @staticmethod
+  def lod2_solid(o):
+    try:
+      solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']['#text']))
+                  for s in o['Building']['lod2Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
+    except TypeError:
+      solid_points = [CityGmlTools._solid_points(CityGmlTools._remove_last_point(s['Polygon']['exterior']['LinearRing']['posList']))
+                  for s in o['Building']['lod2Solid']['Solid']['exterior']['CompositeSurface']['surfaceMember']]
+    return [Surface(Polygon(sp),Polygon(sp)) for sp in solid_points]

imports/geometry/citygml_tools.py

@@ -0,0 +1,22 @@
+import numpy as np
+from imports.geometry.helpers.geometry_helper import GeometryHelper
+
+
+class CityGmlTools:
+
+  @staticmethod
+  def _remove_last_point(points):
+    array = points.split(' ')
+    res = " "
+    return res.join(array[0:len(array) - 3])
+
+  @staticmethod
+  def _solid_points(coordinates) -> np.ndarray:
+    """
+    Solid surface point matrix [[x, y, z],[x, y, z],...]
+    :parameter coordinates: string from file
+    :return: np.ndarray
+    """
+    solid_points = np.fromstring(coordinates, dtype=float, sep=' ')
+    solid_points = GeometryHelper.to_points_matrix(solid_points)
+    return solid_points

imports/geometry/helpers/geometry_helper.py

@@ -293,3 +293,25 @@ class GeometryHelper:
     :return: str
     """
     return GeometryHelper.fuction_to_usage[building_function]
+
+  @staticmethod
+  def to_points_matrix(points):
+    """
+    Transform a point vector into a point matrix
+    :param points: [x, y, z, x, y, z ...]
+    :return: [[x,y,z],[x,y,z]...]
+    """
+    rows = points.size // 3
+    points = points.reshape(rows, 3)
+    return points
+
+  def almost_equal(self, delta_max, v1, v2):
+    """
+    Compare two points and decides if they are almost equal (distance under delta_max)
+    :param delta_max: maximum distance to be considered same point
+    :param v1: [x,y,z]
+    :param v2: [x,y,z]
+    :return: Boolean
+    """
+    delta = self.distance_between_points(v1, v2)
+    return delta <= delta_max

imports/sensors/concordia_energy_consumption.py

@@ -3,61 +3,19 @@ Concordia energy consumption
 SPDX - License - Identifier: LGPL - 3.0 - or -later
 Copyright © 2021 Project Author Guille Gutierrez guillermo.gutierrezmorote@concordia.ca
 """
-import io
-import json
-from pathlib import Path
 import pandas as pd
+from imports.sensors.concordia_file_report import ConcordiaFileReport
 from city_model_structure.attributes.concordia_energy_sensor import ConcordiaEnergySensor
 
 
-class ConcordiaEnergyConsumption:
+class ConcordiaEnergyConsumption(ConcordiaFileReport):
 
   def __init__(self, city, end_point, base_path):
-
+    super().__init__(city, end_point, base_path, 'concordia_energy_db.json')
-    self._buildings = []
-    self._sensors = []
-    self._sensor_point = {}
-    self._city = city
-    self._end_point = end_point
-    self._sensor_database = base_path
-    metadata = True
-    content = False
-    with open(Path(base_path / 'concordia.json').resolve()) as concordia_db:
-      self._sensor_database = json.load(concordia_db)
-
-    for building in self._sensor_database['sensors']:
-      building_name = building['building']
-      for sensor in building['sensors']:
-        self._buildings.append(building_name)
-        self._sensors.append(sensor)
-
-    buffer = ""
-    with open(end_point.resolve()) as data:
-      for line in data:
-        line = ConcordiaEnergyConsumption.clean_line(line)
-        if metadata:
-          fields = line.split(',')
-          if len(fields) > 2:
-            point = fields[0].replace(":", "")
-            key = fields[1]
-            if fields[1] in self._sensors:
-              self._sensor_point[key] = point
-        if "End of Report" in line:
-          content = False
-        if content:
-          line = ConcordiaEnergyConsumption.merge_date_time(line)
-          buffer = buffer + line + '\n'
-        if line is '':
-          metadata = False
-          content = True
-    measures = pd.read_csv(io.StringIO(buffer), sep=',')
-    measures["Date time"] = pd.to_datetime(measures["Date time"])
-    measures = ConcordiaEnergyConsumption.force_format(measures)
-
     for building in city.buildings:
       for i in range(len(self._buildings)):
-        if self._buildings[i] == building.name:
+        if self._buildings[i] == building.name and self._sensors[i] in self._sensor_point:
-          building_measures = [measures["Date time"], measures[self._sensor_point[self._sensors[i]]]]
+          building_measures = [self._measures["Date time"], self._measures[self._sensor_point[self._sensors[i]]]]
           building_headers = ["Date time", "Energy consumption"]
           building_energy_consumption = pd.concat(building_measures, keys=building_headers, axis=1)
           sensor = ConcordiaEnergySensor(self._sensors[i])
@@ -71,25 +29,3 @@ class ConcordiaEnergyConsumption:
             sensor.add_period(building_energy_consumption)
             building.sensors.append(sensor)
 
-  @staticmethod
-  def clean_line(line):
-    return line.replace('"', '').replace('\n', '')
-
-  @staticmethod
-  def merge_date_time(line):
-    fields = line.split(',')
-    date = fields[0]
-    time = fields[1]
-    if '<>' in date:
-      return line.replace(f'{date},{time}', 'Date time')
-    else:
-      date_fields = date.split('/')
-      format_date_time = f'"{int(date_fields[2])}-{int(date_fields[0]):02d}-{int(date_fields[1]):02d} {time}"'
-      return line.replace(f'{date},{time}', format_date_time)
-
-  @staticmethod
-  def force_format(df):
-    for head in df.head():
-      if 'Date time' not in head:
-        df = df.astype({head: 'float64'})
-    return df

imports/sensors/concordia_file_report.py

@@ -0,0 +1,77 @@
+"""
+Concordia file report
+SPDX - License - Identifier: LGPL - 3.0 - or -later
+Copyright © 2021 Project Author Guille Gutierrez guillermo.gutierrezmorote@concordia.ca
+"""
+
+import io
+import json
+from pathlib import Path
+import pandas as pd
+from city_model_structure.attributes.concordia_energy_sensor import ConcordiaEnergySensor
+
+class ConcordiaFileReport:
+  def __init__(self, city, end_point, base_path, db_file):
+
+    self._buildings = []
+    self._sensors = []
+    self._sensor_point = {}
+    self._city = city
+    self._end_point = end_point
+    self._sensor_database = base_path
+    metadata = True
+    content = False
+    with open(Path(base_path / db_file).resolve()) as concordia_db:
+      self._sensor_database = json.load(concordia_db)
+
+    for building in self._sensor_database['sensors']:
+      building_name = building['building']
+      for sensor in building['sensors']:
+        self._buildings.append(building_name)
+        self._sensors.append(sensor)
+
+    buffer = ""
+    with open(end_point.resolve()) as data:
+      for line in data:
+        line = ConcordiaFileReport.clean_line(line)
+        if metadata:
+          fields = line.split(',')
+          if len(fields) > 2:
+            point = fields[0].replace(":", "")
+            key = fields[1]
+            if fields[1] in self._sensors:
+              self._sensor_point[key] = point
+        if "End of Report" in line:
+          content = False
+        if content:
+          line = ConcordiaFileReport.merge_date_time(line)
+          buffer = buffer + line + '\n'
+        if line is '':
+          metadata = False
+          content = True
+    measures = pd.read_csv(io.StringIO(buffer), sep=',')
+    measures["Date time"] = pd.to_datetime(measures["Date time"])
+    self._measures = ConcordiaFileReport.force_format(measures)
+
+  @staticmethod
+  def clean_line(line):
+    return line.replace('"', '').replace('\n', '')
+
+  @staticmethod
+  def merge_date_time(line):
+    fields = line.split(',')
+    date = fields[0]
+    time = fields[1]
+    if '<>' in date:
+      return line.replace(f'{date},{time}', 'Date time')
+    else:
+      date_fields = date.split('/')
+      format_date_time = f'"{int(date_fields[2])}-{int(date_fields[0]):02d}-{int(date_fields[1]):02d} {time}"'
+      return line.replace(f'{date},{time}', format_date_time)
+
+  @staticmethod
+  def force_format(df):
+    for head in df.head():
+      if 'Date time' not in head:
+        df = df.astype({head: 'float64'})
+    return df

imports/sensors_factory.py

@@ -20,6 +20,9 @@ class SensorsFactory:
   def _cec(self):
     ConcordiaEnergyConsumption(self._city, self._end_point, self._base_path)
 
+  def _cgf(self):
+    ConcordiaGasFlow(self._city, self._end_point, self._base_path)
+
   def enrich(self):
     """
     Enrich the city with the usages information

non_functional_tests/test_geometry_factory.py

@@ -29,7 +29,7 @@ class TestGeometryFactory(TestCase):
     return self._city
 
   def _get_obj(self, file):
-    # todo: solve the incongruences between city and city_debug
+    # todo: solve the incongruities between city and city_debug
     file_path = (self._example_path / file).resolve()
     self._city = GeometryFactory('obj', file_path)._city_debug
     self.assertIsNotNone(self._city, 'city is none')

non_functional_tests/test_sensors_factory.py

@@ -49,3 +49,4 @@ class TestSensorsFactory(TestCase):
         sensor.add_period(update)
         row = sensor.measures.loc[sensor.measures["Date time"] == '2020-01-19 23:55:00']['Energy consumption'].iloc[0]
         self.assertTrue(f'{row}' == '12345.0')
+