a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Provide transform when joining data

Browse Source
This commit is contained in:
Tom Russell 2018-08-01 17:05:02 +01:00
parent 847eb75a70
commit 69bb6c5790
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
etl/3_join-data.py
View File

@ -18,16 +18,17 @@ def main(source_file, config_path, transform_config_path):
"""Load config, read files and save features to the database """Load config, read files and save features to the database
""" """
conf = read_config(config_path) conf = read_config(config_path)
transform_config = read_config(transform_config_path)
data_mapping = [ data_mapping = [
# from_fieldname, to_fieldname, mapping(old_val, source_val)->new_val # from_fieldname, to_fieldname, mapping(old_val, source_val)->new_val
(from_, to_, eval(transform)) (from_, to_, eval(transform))
for from_, to_, transform in read_config(transform_config_path) for from_, to_, transform in transform_config['mapping']
] ]
dbconf = conf['database'] dbconf = conf['database']
conn = psycopg2.connect(**dbconf) conn = psycopg2.connect(**dbconf)
with fiona.open(source_file, 'r') as source: with fiona.open(source_file, 'r') as source:
epsg_code = int(source.crs['init'].replace("epsg:","")) epsg_code = transform_config['crs']
with conn.cursor() as cur: with conn.cursor() as cur:
for feature in source: for feature in source:
@ -47,9 +48,9 @@ def save_data(cur, props, data_conf, geometry_id):
cur.execute( cur.execute(
"""SELECT building_id, building_doc FROM buildings """SELECT building_id, building_doc FROM buildings
WHERE WHERE
geometry_id = % geometry_id = %s
""", ( """, (
geometry_id geometry_id,
) )
) )
building = cur.fetchone() building = cur.fetchone()
@ -81,7 +82,7 @@ def save_data(cur, props, data_conf, geometry_id):
WHERE WHERE
building_id = %s building_id = %s
""", ( """, (
doc, json.dumps(doc),
building_id building_id
) )
) )
@ -94,7 +95,10 @@ def find_geom(cur, feature, epsg_code):
"""SELECT geometry_id FROM geometries """SELECT geometry_id FROM geometries
WHERE WHERE
ST_Intersects( ST_Intersects(
ST_Transform(
ST_SetSRID(%s::geometry, %s), ST_SetSRID(%s::geometry, %s),
3857
),
geometry_geom geometry_geom
) )
""", ( """, (
@ -131,7 +135,7 @@ def read_config(config_path):
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) != 3: if len(sys.argv) != 4:
print( print(
"Usage: {} ./path/to/source/file.csv ./path/to/dbconfig.json ./path/to/mapping.json".format( "Usage: {} ./path/to/source/file.csv ./path/to/dbconfig.json ./path/to/mapping.json".format(
os.path.basename(__file__) os.path.basename(__file__)