From 420ea0e8d8db8c12d2de27909c1a9cd086e0ceea Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Thu, 17 Mar 2022 11:32:41 +0100 Subject: [PATCH 01/17] add type as attribute in data table --- .../data_extraction/data_extraction.py | 10 ++++++++-- .../sql/data_extraction/create_surrounding_table.sql | 1 + .../sql/data_extraction/upload_query.sql | 1 + .../sql/data_extraction/upload_template.sql | 1 + .../sql/setup/create_required_minimum_tables.sql | 1 + 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/osm_element_vectorisation/data_extraction/data_extraction.py b/src/osm_element_vectorisation/data_extraction/data_extraction.py index a897d97..a979f0d 100644 --- a/src/osm_element_vectorisation/data_extraction/data_extraction.py +++ b/src/osm_element_vectorisation/data_extraction/data_extraction.py @@ -195,8 +195,13 @@ def parse_response_list(settings: ActivatedSettings, response_list: list): props = feature["properties"] if "way" in props["@osmId"]: ident = int(re.sub("way/", "", props["@osmId"])) - else: - ident = -int(re.sub("relation/", "", props["@osmId"])) + obj_type = "way" + if "node" in props["@osmId"]: + ident = int(re.sub("node/", "", props["@osmId"])) + obj_type = "node" + elif "relation" in props["@osmId"]: + ident = int(re.sub("relation/", "", props["@osmId"])) + obj_type = "relation" geom = json.dumps(feature["geometry"]) # for each property, check if it is a primary tag as defined by the user @@ -204,6 +209,7 @@ def parse_response_list(settings: ActivatedSettings, response_list: list): if (str(pkey) + "=*" in tags) or (str(pkey) + "=" + str(pval) in tags): upload_obj = ( ident, + obj_type, pkey, pval, parse_hstore(props, pkey), diff --git a/src/osm_element_vectorisation/sql/data_extraction/create_surrounding_table.sql b/src/osm_element_vectorisation/sql/data_extraction/create_surrounding_table.sql index d044a5b..8e0c001 100644 --- a/src/osm_element_vectorisation/sql/data_extraction/create_surrounding_table.sql +++ b/src/osm_element_vectorisation/sql/data_extraction/create_surrounding_table.sql @@ -1,6 +1,7 @@ CREATE TABLE IF NOT EXISTS {surrounding_table}( id serial PRIMARY KEY, osm_id bigint, + type TEXT, primkey TEXT, primval TEXT, othertags hstore, diff --git a/src/osm_element_vectorisation/sql/data_extraction/upload_query.sql b/src/osm_element_vectorisation/sql/data_extraction/upload_query.sql index c607045..41a0698 100644 --- a/src/osm_element_vectorisation/sql/data_extraction/upload_query.sql +++ b/src/osm_element_vectorisation/sql/data_extraction/upload_query.sql @@ -2,6 +2,7 @@ INSERT INTO {0}( osm_id, + type, primkey, primval, othertags, diff --git a/src/osm_element_vectorisation/sql/data_extraction/upload_template.sql b/src/osm_element_vectorisation/sql/data_extraction/upload_template.sql index 0d3f2cf..ba17fa6 100644 --- a/src/osm_element_vectorisation/sql/data_extraction/upload_template.sql +++ b/src/osm_element_vectorisation/sql/data_extraction/upload_template.sql @@ -4,6 +4,7 @@ %s, %s, %s, + %s, {make_valid_geog_func}( st_geomfromgeojson( %s diff --git a/src/osm_element_vectorisation/sql/setup/create_required_minimum_tables.sql b/src/osm_element_vectorisation/sql/setup/create_required_minimum_tables.sql index b997406..e7b22f3 100644 --- a/src/osm_element_vectorisation/sql/setup/create_required_minimum_tables.sql +++ b/src/osm_element_vectorisation/sql/setup/create_required_minimum_tables.sql @@ -1,6 +1,7 @@ CREATE TABLE IF NOT EXISTS {data_table_name} ( id serial PRIMARY KEY, osm_id bigint, + type TEXT, primkey TEXT, primval TEXT, othertags hstore, -- GitLab From 12823b19111669086cd5799c28201dbc4e9390f9 Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Mon, 21 Mar 2022 10:19:01 +0100 Subject: [PATCH 02/17] consider osm type for download by id --- .../data_extraction/data_extraction.py | 15 ++++--------- .../sql/data_extraction/check_presence.sql | 21 +++++++++++++------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/osm_element_vectorisation/data_extraction/data_extraction.py b/src/osm_element_vectorisation/data_extraction/data_extraction.py index a979f0d..78932a9 100644 --- a/src/osm_element_vectorisation/data_extraction/data_extraction.py +++ b/src/osm_element_vectorisation/data_extraction/data_extraction.py @@ -62,10 +62,8 @@ def download_by_id(settings: ActivatedSettings): ): logging.debug("Downloading the following osm_ids: %s", id_sub_list) - way_ids, rel_ids = convert_ids(id_list) - ohsome_filter = get_download_filter( - way_ids=way_ids, rel_ids=rel_ids, tags=tag_string - ) + way_ids = convert_ids(id_list) + ohsome_filter = get_download_filter(way_ids=way_ids, tags=tag_string) insert_ids.extend( request_n_upload( @@ -75,7 +73,6 @@ def download_by_id(settings: ActivatedSettings): table_name="{data_table_name}", ) ) - return insert_ids @@ -105,7 +102,7 @@ def download_by_bbox(settings: ActivatedSettings): return insert_ids -def get_download_filter(way_ids: str, rel_ids: str, tags: str): +def get_download_filter(way_ids: str, tags: str): """Place givens in an ohsome filter. :param way_ids: @@ -113,11 +110,7 @@ def get_download_filter(way_ids: str, rel_ids: str, tags: str): :param tags: :return: """ - ohsome_filter = ( - f"({tags}) and geometry:polygon " - f"and ((id:({way_ids}) and type:way) or " - f"(id:({rel_ids}) and type:relation))" - ) + ohsome_filter = f"({tags}) and geometry:polygon " f"and (id:({way_ids}))" return ohsome_filter diff --git a/src/osm_element_vectorisation/sql/data_extraction/check_presence.sql b/src/osm_element_vectorisation/sql/data_extraction/check_presence.sql index caa9651..a544d8f 100644 --- a/src/osm_element_vectorisation/sql/data_extraction/check_presence.sql +++ b/src/osm_element_vectorisation/sql/data_extraction/check_presence.sql @@ -1,16 +1,25 @@ -WITH ids AS ( +WITH +unnested_ids as( +SELECT +unnest(ARRAY[split_part(unnest(array[%(searchIds)s]), '/', 1)]) as type, +unnest(ARRAY[split_part(unnest(array[%(searchIds)s]), '/', 2)]) as id +), +ids AS ( SELECT - reqId, + unnested_ids.id, + unnested_ids.type, ARRAY_AGG(tstamp_download) AS tss FROM - UNNEST(%(searchIds)s) AS reqId + unnested_ids LEFT JOIN {data_table_name} td ON - reqId = td.osm_id + unnested_ids.id::bigint = td.osm_id AND + unnested_ids.type = td.type GROUP BY - reqId + unnested_ids.id, + unnested_ids.type ) SELECT - reqId + id, type FROM ids WHERE -- GitLab From c0400aa4d0fa9f682ba87b67def479264009d9bf Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Mon, 21 Mar 2022 10:36:03 +0100 Subject: [PATCH 03/17] remove assignment of negative osm ids --- .../data_extraction/data_extraction.py | 26 ++++++------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/osm_element_vectorisation/data_extraction/data_extraction.py b/src/osm_element_vectorisation/data_extraction/data_extraction.py index 78932a9..ac46f90 100644 --- a/src/osm_element_vectorisation/data_extraction/data_extraction.py +++ b/src/osm_element_vectorisation/data_extraction/data_extraction.py @@ -62,8 +62,8 @@ def download_by_id(settings: ActivatedSettings): ): logging.debug("Downloading the following osm_ids: %s", id_sub_list) - way_ids = convert_ids(id_list) - ohsome_filter = get_download_filter(way_ids=way_ids, tags=tag_string) + osm_ids = convert_ids(id_list) + ohsome_filter = get_download_filter(osm_ids=osm_ids, tags=tag_string) insert_ids.extend( request_n_upload( @@ -102,35 +102,25 @@ def download_by_bbox(settings: ActivatedSettings): return insert_ids -def get_download_filter(way_ids: str, tags: str): +def get_download_filter(osm_ids: str, tags: str): """Place givens in an ohsome filter. - :param way_ids: - :param rel_ids: + :param osm_ids: :param tags: :return: """ - ohsome_filter = f"({tags}) and geometry:polygon " f"and (id:({way_ids}))" + ohsome_filter = f"({tags}) and geometry:polygon " f"and (id:({osm_ids}))" return ohsome_filter def convert_ids(id_list): - """Convert positive and negative ids to way and relation filter. + """Convert id list to string for ohsome filter. :param id_list: :return: """ - way_ids = [0] - rel_ids = [0] - for num in id_list: - if num > 0: - way_ids.append(num) - else: - rel_ids.append(-num) - - way_ids = ",".join([str(x) for x in way_ids]) - rel_ids = ",".join([str(x) for x in rel_ids]) - return way_ids, rel_ids + way_ids = ",".join([str(x) for x in id_list]) + return way_ids def request_n_upload(settings, ohsome_filter, bbox, table_name): -- GitLab From a1fedbd5c9eafef694b020beef578eca069f404b Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Thu, 24 Mar 2022 12:59:48 +0100 Subject: [PATCH 04/17] adjust fs preprocessing to new table structure --- .../preprocessing/preprocessing.py | 27 ++++++++++--------- .../resources/sql/preprocessing/get_ids.sql | 1 + 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py b/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py index ab71541..0f3935f 100644 --- a/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py +++ b/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py @@ -40,22 +40,23 @@ def get_contribs(settings: ActivatedSettings): ) # get contributions to target objects - ids = settings.db_parameters.ex_select(read_text(sql_files, "get_ids.sql")) - ids = [item[0] for item in ids] + ids_types = settings.db_parameters.ex_select(read_text(sql_files, "get_ids.sql")) + ids = [item[1] for item in ids_types] # split id list in chunks of length n (maybe higher?) n_length = 1000 - ids = [ids[i : i + n_length] for i in range(0, len(ids), n_length)] - if ids: - for lst in tqdm(ids, desc="ID chunks for contribution extraction", leave=False): - - curr_ids = [ - "way/" + str(osmid) if osmid > 0 else "relation/" + str(abs(osmid)) - for osmid in lst - ] - + ids_types_list = [ + (ids_types[0][i : i + n_length], ids_types[1][i : i + n_length]) + for i in range(0, len(ids_types), n_length) + ] + if ids_types: + for lst in tqdm( + ids_types_list, desc="ID chunks for contribution extraction", leave=False + ): + + curr_ids = [str(osmid[0]) + "/" + str(osmid[1]) for osmid in lst] ohsome_filter = "id:(" + ", ".join(curr_ids) + ")" - response = __query_contribs(settings, lst, ohsome_filter, endpoint, time) + response = __query_contribs(settings, ids, ohsome_filter, endpoint, time) contribs_list = __process_contrib_result( response, settings.processing.timestamp ) @@ -161,7 +162,7 @@ def __process_contrib_result(response: list, target_timestamp: datetime): for res in response: for feature in res["features"]: props = feature["properties"] - ident = processing.ids_from_ohsome(props) + ident = processing.ids_from_ohsome(props)[0] timestamp = props["@timestamp"] cs_id = props["@contributionChangesetId"] for contrib_type in types: diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql index 6dee064..4a7878d 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql @@ -1,4 +1,5 @@ SELECT + type, osm_id FROM {data_table_name} td -- GitLab From 71a5443f3f4e5e6e4c6948fa1d4991ca4945c469 Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Thu, 24 Mar 2022 13:06:48 +0100 Subject: [PATCH 05/17] fix typo --- .../fs_calculation/element/calc_element.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osm_element_vectorisation/fs_calculation/element/calc_element.py b/src/osm_element_vectorisation/fs_calculation/element/calc_element.py index eca099f..0239e94 100644 --- a/src/osm_element_vectorisation/fs_calculation/element/calc_element.py +++ b/src/osm_element_vectorisation/fs_calculation/element/calc_element.py @@ -43,7 +43,7 @@ def calc_element(settings: ActivatedSettings): settings.db_parameters.ex_iud(read_text(element, "nchanges.sql")) settings.db_parameters.ex_iud(read_text(element, "time.sql")) - if settings.keys.sentinelhub_key and ("sentilenhub" in sys.modules): + if settings.keys.sentinelhub_key and ("sentinelhub" in sys.modules): logging.info("Request remote sensing attributes") calc_rs(settings) -- GitLab From d6919bd6812483d6bc184bda84e0f169906e917e Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Thu, 24 Mar 2022 14:04:15 +0100 Subject: [PATCH 06/17] adapt config description to table structure with separate columns for type and id --- doc/config.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/config.md b/doc/config.md index 1e043fd..6d148d2 100644 --- a/doc/config.md +++ b/doc/config.md @@ -57,9 +57,9 @@ Here you can see an example configuration that can be adapted to your needs with - **timestamp** (mandatory) - Timestamp in relation to which the data will be analysed in ISO-8601 - **limit_ids** (optional; default: empty) - - OSM-IDs to which the extraction should be limited. This is useful if you want to only analyse some objects within the aoi. Positive IDs are interpreted as OSM-Way IDs, negative as OSM-Relation IDs. Choose one of the following formats or leave empty to query and analyse all data within the aoi: - - a JSON list of integers with IDs (e.g. [12376,-34289]) - - a path to a csv file containing the ids as "/path/to/file.csv" + - OSM-IDs to which the extraction should be limited. This is useful if you want to only analyse some objects within the aoi. Choose one of the following formats or leave empty to query and analyse all data within the aoi: + - a JSON list of strings with IDs (e.g. ["way/12376","relation/34289"]) + - a path to a csv file containing the types and ids as "/path/to/file.csv" - **geom_type** (optional; default: "polygon") - Geometry type for which indicators will be calculated. Possible values: `"point"`, `"line"`, `"polygon"` - **target_database_conn** (optional; default: docker database) -- GitLab From cc6c7cfd409da9cc9679170e385ac0a4fe427c72 Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Tue, 29 Mar 2022 11:36:28 +0200 Subject: [PATCH 07/17] add column for osm type --- .../resources/sql/export/get_geodata.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql b/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql index a945d60..2bc9dbf 100644 --- a/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql +++ b/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql @@ -1,6 +1,7 @@ SELECT id, osm_id, + type, geom, note FROM -- GitLab From 828490efbf66079b6535db9403c623743b6d3aba Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Tue, 29 Mar 2022 15:46:33 +0200 Subject: [PATCH 08/17] add new osm ids to docker and cli --- src/osm_element_vectorisation/config/cli.py | 2 +- vectorisation_docker.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osm_element_vectorisation/config/cli.py b/src/osm_element_vectorisation/config/cli.py index da6fa4d..525625d 100644 --- a/src/osm_element_vectorisation/config/cli.py +++ b/src/osm_element_vectorisation/config/cli.py @@ -55,7 +55,7 @@ def parse_cl_args(): ) id_mutex_group = id_group.add_mutually_exclusive_group() id_mutex_group.add_argument( - "--ids", metavar="id", nargs="+", help="A list of ids.", type=int + "--ids", metavar="id", nargs="+", help="A list of ids.", type=str ) id_mutex_group.add_argument( "--id_file", diff --git a/vectorisation_docker.sh b/vectorisation_docker.sh index fed6404..86bc1ed 100755 --- a/vectorisation_docker.sh +++ b/vectorisation_docker.sh @@ -3,7 +3,7 @@ case $1 in repex) docker-compose --project-directory . -f Docker/docker-compose.yaml run import "$1" - docker-compose --project-directory . -f Docker/docker-compose.yaml run vectorisation Docker/setup/repex_data/repex_config.json --bbox 1.2045802,9.5105353,1.2078996,9.506396 --timestamp 2022-01-15 --ids 1021101718 --data_schema "$1" + docker-compose --project-directory . -f Docker/docker-compose.yaml run vectorisation Docker/setup/repex_data/repex_config.json --bbox 1.2045802,9.5105353,1.2078996,9.506396 --timestamp 2022-01-15 --ids way/1021101718 --data_schema "$1" docker-compose --project-directory . -f Docker/docker-compose.yaml down ;; benchmark) @@ -36,7 +36,7 @@ case $1 in #docker-compose --project-directory . -f Docker/docker-compose.yaml run --entrypoint bash vectorisation # download by id - docker-compose --project-directory . -f Docker/docker-compose.yaml run vectorisation Docker/setup/repex_data/repex_config.json --bbox 1.2045802,9.5105353,1.2078996,9.506396 --timestamp 2022-01-15 --ids 1021101718 --data_schema "$1" + docker-compose --project-directory . -f Docker/docker-compose.yaml run vectorisation Docker/setup/repex_data/repex_config.json --bbox 1.2045802,9.5105353,1.2078996,9.506396 --timestamp 2022-01-15 --ids way/1021101718 --data_schema "$1" # download by bbox #docker-compose --project-directory . -f Docker/docker-compose.yaml run vectorisation Docker/setup/repex_data/repex_config.json --bbox 1.2045802,9.5105353,1.2078996,9.506396 --timestamp 2022-01-15 --data_schema "$1" -- GitLab From b3e226165c92c53e5e85d892219e769a517feddf Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Tue, 29 Mar 2022 15:47:47 +0200 Subject: [PATCH 09/17] fix fs preprocessing for single id input --- .../fs_calculation/preprocessing/preprocessing.py | 11 +++++------ .../resources/sql/preprocessing/get_ids.sql | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py b/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py index 5e9dde2..0da337b 100644 --- a/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py +++ b/src/osm_element_vectorisation/fs_calculation/preprocessing/preprocessing.py @@ -41,21 +41,20 @@ def get_contribs(settings: ActivatedSettings): # get contributions to target objects ids_types = settings.db_parameters.ex_select(read_text(sql_files, "get_ids.sql")) - ids = [item[1] for item in ids_types] + ids_types = [item[0] for item in ids_types] # split id list in chunks of length n (maybe higher?) n_length = 1000 ids_types_list = [ - (ids_types[0][i : i + n_length], ids_types[1][i : i + n_length]) - for i in range(0, len(ids_types), n_length) + (ids_types[i : i + n_length]) for i in range(0, len(ids_types), n_length) ] + if ids_types: for lst in tqdm( ids_types_list, desc="ID chunks for contribution extraction", leave=False ): - - curr_ids = [str(osmid[0]) + "/" + str(osmid[1]) for osmid in lst] - ohsome_filter = "id:(" + ", ".join(curr_ids) + ")" + ids = [int(i.split("/")[1]) for i in lst] + ohsome_filter = "id:(" + ", ".join(ids_types) + ")" response = __query_contribs(settings, ids, ohsome_filter, endpoint, time) contribs_list = __process_contrib_result( response, settings.processing.timestamp diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql index 4a7878d..87f54cc 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql @@ -1,6 +1,5 @@ SELECT - type, - osm_id + concat(type, '/', osm_id) FROM {data_table_name} td LEFT JOIN {edits_table_name} drie ON -- GitLab From a7ed1e9dde996730dfdbadf3f30c4bd5e54ab70e Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Wed, 6 Apr 2022 10:12:40 +0200 Subject: [PATCH 10/17] add column for osm type --- .../resources/sql/preprocessing/fs_setup.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/fs_setup.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/fs_setup.sql index e1fe277..0663939 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/fs_setup.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/fs_setup.sql @@ -185,7 +185,7 @@ sauce_agg AS ( ) SELECT itn.ref_id, - osm_id<0 AS relation, + type, concat(dtn.primkey, '=', dtn.primval) AS primtag, cl.level1 AS corine_class, 3 * validity_issues_geom::integer + 2 * validity_issues_geog::integer + validity_issues_postprocess::integer + geom_change_postprocess::integer AS invalidity, -- GitLab From ed967ee33d90335111c0204fc5fec7164cafdbb6 Mon Sep 17 00:00:00 2001 From: Leonie Groesschen Date: Wed, 6 Apr 2022 11:23:28 +0200 Subject: [PATCH 11/17] change column 'relation' to 'type' --- .../resources/misc/predictors_dtypes.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json index f88c1f6..a5d6f65 100644 --- a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json +++ b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json @@ -1,6 +1,6 @@ { "ref_id": "int64", - "relation": "bool", + "type": "str", "primtag": "str", "corine_class": "str", "invalidity": "int64", -- GitLab From 2cb54c0c1d07983d130908281be44573b2448392 Mon Sep 17 00:00:00 2001 From: Moritz Schott Date: Tue, 12 Apr 2022 13:09:48 +0200 Subject: [PATCH 12/17] fix osm_type transversion --- src/osm_element_vectorisation/config/cli.py | 4 +- .../config/config.py | 15 +++--- .../config/db_parameters.py | 9 +--- .../config/ohsome.py | 3 +- .../config/processing.py | 25 ++++++++-- .../data_extraction/data_extraction.py | 41 +++++++--------- .../db_setup/db_setup.py | 17 +++++++ .../indicators/preprocessing/preprocessing.py | 47 +++++++++++-------- .../resources/misc/predictors_dtypes.json | 2 +- .../sql/data_extraction/check_presence.sql | 40 +++++++--------- .../create_surrounding_table.sql | 9 +++- .../sql/data_extraction/upload_query.sql | 4 +- .../resources/sql/export/get_geodata.sql | 2 +- .../sql/preprocessing/contibs_upload.sql | 17 +++---- .../resources/sql/preprocessing/get_ids.sql | 4 +- .../preprocessing/indicator_table_setup.sql | 2 +- .../resources/sql/setup/check_extensions.sql | 1 + .../setup/create_required_minimum_tables.sql | 9 +++- .../resources/sql/setup/queries/check_ts.sql | 7 +++ .../sql/setup/queries/get_buffer_query.sql | 3 +- 20 files changed, 151 insertions(+), 110 deletions(-) create mode 100644 src/osm_element_vectorisation/resources/sql/setup/queries/check_ts.sql diff --git a/src/osm_element_vectorisation/config/cli.py b/src/osm_element_vectorisation/config/cli.py index 525625d..3f0e7a5 100644 --- a/src/osm_element_vectorisation/config/cli.py +++ b/src/osm_element_vectorisation/config/cli.py @@ -54,9 +54,7 @@ def parse_cl_args(): "config `limit_ids`. You can choose between two formats:", ) id_mutex_group = id_group.add_mutually_exclusive_group() - id_mutex_group.add_argument( - "--ids", metavar="id", nargs="+", help="A list of ids.", type=str - ) + id_mutex_group.add_argument("--ids", metavar="id", nargs="+", help="A list of ids.") id_mutex_group.add_argument( "--id_file", metavar="/path/to/file.csv", diff --git a/src/osm_element_vectorisation/config/config.py b/src/osm_element_vectorisation/config/config.py index 15f68db..a0589bc 100644 --- a/src/osm_element_vectorisation/config/config.py +++ b/src/osm_element_vectorisation/config/config.py @@ -6,7 +6,7 @@ import pathlib import sys from datetime import datetime from json import JSONDecodeError -from typing import List, Optional +from typing import List, Optional, Tuple from ohsome import OhsomeClient from psycopg2 import OperationalError @@ -86,8 +86,8 @@ def __parse_optional_args(config_dict: dict, processing: Processing) -> dict: """ out = {} - processing.limit_ids = __get_limit_ids( - config_dict.get("limit_ids", processing.limit_ids) + processing.limit_ids = ( + __get_limit_ids(config_dict.get("limit_ids", None)) or processing.limit_ids ) processing.geom_type = GeomTypes(config_dict.get("geom_type", processing.geom_type)) processing.stop_after = ProcessingSteps( @@ -129,7 +129,7 @@ def __parse_optional_args(config_dict: dict, processing: Processing) -> dict: return out -def __get_limit_ids(limit_ids) -> Optional[List[int]]: +def __get_limit_ids(limit_ids) -> Optional[List[Tuple[str, int]]]: """Convert parameter to id list. :param limit_ids: @@ -145,12 +145,13 @@ def __get_limit_ids(limit_ids) -> Optional[List[int]]: reader = csv.reader(file) data = list(reader) - id_list = [ - int(item) for sublist in data for item in sublist - ] # unlist & convert to int + id_list = [str(item) for sublist in data for item in sublist] else: logging.error("did not understand input for ids") sys.exit(1) + + id_list = [(x.split("/")[0], int(x.split("/")[1])) for x in id_list] + return id_list diff --git a/src/osm_element_vectorisation/config/db_parameters.py b/src/osm_element_vectorisation/config/db_parameters.py index c57eeb6..530e2a2 100644 --- a/src/osm_element_vectorisation/config/db_parameters.py +++ b/src/osm_element_vectorisation/config/db_parameters.py @@ -117,16 +117,11 @@ class DBConnectionManager: self.target_db.putconn(conn) return res - def get_buffer_df( - self, ref_id_list: List[int] = None, osm_id_list: List[int] = None - ) -> geopandas.GeoDataFrame: + def get_buffer_df(self, ref_id_list: List[int]) -> geopandas.GeoDataFrame: """Get GDF of buffer geoms with given ref_id or osm_id.""" - osm_id_list = osm_id_list or [] - ref_id_list = ref_id_list or [] - buf = self.get_db_geopandas( query=read_text(queries, "get_buffer_query.sql"), - params={"ref_id_list": ref_id_list, "osm_id_list": osm_id_list}, + params={"ref_id_list": ref_id_list}, ) return buf diff --git a/src/osm_element_vectorisation/config/ohsome.py b/src/osm_element_vectorisation/config/ohsome.py index 123ca82..f7accee 100644 --- a/src/osm_element_vectorisation/config/ohsome.py +++ b/src/osm_element_vectorisation/config/ohsome.py @@ -1,6 +1,7 @@ """ohsome-py client collection.""" import logging import math +import pathlib import time from datetime import datetime from typing import List @@ -26,7 +27,7 @@ class OhsomeClients: log_arg = {} for handler in logging.getLogger().handlers: if hasattr(handler, "baseFilename"): - log_arg["log_dir"] = handler.baseFilename + log_arg["log_dir"] = pathlib.Path(handler.baseFilename).parent break self.main = OhsomeClient(base_api_url=ohsome_url, **log_arg) diff --git a/src/osm_element_vectorisation/config/processing.py b/src/osm_element_vectorisation/config/processing.py index 2ed219f..6314165 100644 --- a/src/osm_element_vectorisation/config/processing.py +++ b/src/osm_element_vectorisation/config/processing.py @@ -1,11 +1,13 @@ """Class for processing settings.""" # pylint: disable=too-many-arguments # pylint: disable=too-many-instance-attributes +import logging import os import pathlib import re from datetime import datetime, timedelta from logging import INFO +from typing import List, Tuple import pandas from pandas import IntervalIndex @@ -20,7 +22,7 @@ class Processing: self, aoi: str, timestamp: datetime, - limit_ids: list = None, + limit_ids: List[Tuple[str, int]] = None, geom_type=GeomTypes.POLYGON, stop_after=ProcessingSteps.EXPORT, comment: str = None, @@ -78,20 +80,33 @@ class Processing: return time_range -def ids_from_ohsome(props: dict): +def ids_from_ohsome(props: dict) -> Tuple[int, str]: """Convert ohsome props to id and object type. :param props: :return: """ - if "way" in props["@osmId"]: - ident = int(re.sub("way/", "", props["@osmId"])) - obj_type = "way" if "node" in props["@osmId"]: ident = int(re.sub("node/", "", props["@osmId"])) obj_type = "node" + elif "way" in props["@osmId"]: + ident = int(re.sub("way/", "", props["@osmId"])) + obj_type = "way" elif "relation" in props["@osmId"]: ident = int(re.sub("relation/", "", props["@osmId"])) obj_type = "relation" + else: + logging.error("Unknown osm type in response. The response may be corrupt.") + return -1, "unknown" return ident, obj_type + + +def ids_to_filter(id_list: List[Tuple[str, int]]) -> str: + """Convert positive and negative ids to way and relation filter. + + :param id_list: + :return: + """ + osm_ids = ",".join([x[0] + "/" + str(x[1]) for x in id_list]) + return osm_ids diff --git a/src/osm_element_vectorisation/data_extraction/data_extraction.py b/src/osm_element_vectorisation/data_extraction/data_extraction.py index 223e2a2..8266058 100644 --- a/src/osm_element_vectorisation/data_extraction/data_extraction.py +++ b/src/osm_element_vectorisation/data_extraction/data_extraction.py @@ -10,6 +10,7 @@ from tqdm import tqdm import osm_element_vectorisation.resources.sql.data_extraction as sql_files from osm_element_vectorisation.config import processing from osm_element_vectorisation.config.db_parameters import DBConnectionManager +from osm_element_vectorisation.config.processing import ids_to_filter from osm_element_vectorisation.config.settings import ActivatedSettings @@ -34,27 +35,25 @@ def download_by_id(settings: ActivatedSettings) -> List[int]: :return: list of inserted ids """ - cleaned_ids = settings.db_parameters.ex_select( - read_text(sql_files, "check_presence.sql"), - { - "searchIds": settings.processing.limit_ids, - "ts": settings.processing.timestamp, - }, + cleaned_ids = settings.db_parameters.ex_sql_batch_upload( + query=read_text(sql_files, "check_presence.sql"), + value_list=settings.processing.limit_ids, + template="(%s::text, %s::bigint)", + fetch=True, ) - id_list = [int(item[0]) for item in cleaned_ids] insert_ids = [] # anything to be downloaded? - if id_list: + if cleaned_ids: tag_string = settings.db_parameters.get_tags_filter() - id_list.sort() + cleaned_ids.sort() # splitting the idlist here may not be necessary # if the server accepts large post requests id_super_list = [] n_steps = 1000 - for i in range(0, len(id_list), n_steps): - id_super_list.append(id_list[i : i + n_steps]) + for i in range(0, len(cleaned_ids), n_steps): + id_super_list.append(cleaned_ids[i : i + n_steps]) for id_sub_list in tqdm( id_super_list, @@ -63,7 +62,7 @@ def download_by_id(settings: ActivatedSettings) -> List[int]: ): logging.debug("Downloading the following osm_ids: %s", id_sub_list) - osm_ids = __convert_ids(id_list) + osm_ids = ids_to_filter(id_sub_list) ohsome_filter = __get_download_filter(osm_ids=osm_ids, tags=tag_string) insert_ids.extend( @@ -114,16 +113,6 @@ def __get_download_filter(osm_ids: str, tags: str) -> str: return ohsome_filter -def __convert_ids(id_list) -> Tuple[str, str]: - """Convert positive and negative ids to way and relation filter. - - :param id_list: - :return: - """ - osm_ids = ",".join([str(x) for x in id_list]) - return osm_ids - - def __request_n_upload(settings, ohsome_filter, bbox, table_name) -> List[int]: """Call functions to request data and upload into database.""" timestamp = settings.processing.timestamp @@ -150,7 +139,9 @@ def __request_n_upload(settings, ohsome_filter, bbox, table_name) -> List[int]: def __parse_response_list( settings: ActivatedSettings, response_list: list -) -> List[Tuple[int, str, str, Optional[dict], datetime.datetime, str, str, int, str]]: +) -> List[ + Tuple[int, str, str, str, Optional[dict], datetime.datetime, str, str, int, str] +]: """Convert responses to upload data. :param settings: @@ -205,7 +196,9 @@ def __parse_hstore(props: dict, pkey: str) -> Optional[dict]: return hstore -def postprocess_target(db_parameters: DBConnectionManager, insert_ids: list) -> None: +def postprocess_target( + db_parameters: DBConnectionManager, insert_ids: List[int] +) -> None: """Run SQL script processing target data.""" # target data processing # this has to be done before deletion otherwise diff --git a/src/osm_element_vectorisation/db_setup/db_setup.py b/src/osm_element_vectorisation/db_setup/db_setup.py index c13ea3c..e0db28d 100644 --- a/src/osm_element_vectorisation/db_setup/db_setup.py +++ b/src/osm_element_vectorisation/db_setup/db_setup.py @@ -46,6 +46,8 @@ def db_setup(settings: Settings) -> ActivatedSettings: read_text(setup, "create_required_minimum_tables.sql") ) + __check_ts(settings) + logging.info("DB Setup is Complete!") return ActivatedSettings(settings, ac_db_params) @@ -164,3 +166,18 @@ def tags_upload(db_parameters: DBConnectionManager) -> None: logging.error("Tags table could not be queried") logging.error(exception) raise exception + + +def __check_ts(settings: Settings) -> None: + exists = settings.db_parameters.ex_select( + read_text(queries, "check_ts.sql"), + {"ts": settings.processing.timestamp}, + ) + if exists: + content = ( + "Only one timestamp per schema is allowed. The given timestamp is " + "different from the data already uploaded." + ) + ex = ValueError(content) + logging.exception(content, exc_info=ex) + raise ex diff --git a/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py b/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py index ca27a82..2285ef0 100644 --- a/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py +++ b/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py @@ -3,12 +3,14 @@ import logging import re from datetime import datetime from importlib.resources import read_text +from typing import List from psycopg2 import sql from tqdm import tqdm from osm_element_vectorisation.config import processing from osm_element_vectorisation.config.db_parameters import ActivatedDBConnectionManager +from osm_element_vectorisation.config.processing import ids_to_filter from osm_element_vectorisation.config.settings import ActivatedSettings from osm_element_vectorisation.resources.sql import preprocessing as sql_files @@ -40,22 +42,22 @@ def get_contribs(settings: ActivatedSettings): ) # get contributions to target objects - ids_types = settings.db_parameters.ex_select(read_text(sql_files, "get_ids.sql")) - ids_types = [item[0] for item in ids_types] + ids = settings.db_parameters.ex_select(read_text(sql_files, "get_ids.sql")) - # split id list in chunks of length n (maybe higher?) - n_length = 1000 - ids_types_list = [ - (ids_types[i : i + n_length]) for i in range(0, len(ids_types), n_length) - ] + if ids: + # split id list in chunks of length n (maybe higher?) + n_length = 1000 + ids_types_list = [(ids[i : i + n_length]) for i in range(0, len(ids), n_length)] - if ids_types: for lst in tqdm( ids_types_list, desc="ID chunks for contribution extraction", leave=False ): - ids = [int(i.split("/")[1]) for i in lst] - ohsome_filter = "id:(" + ", ".join(ids_types) + ")" - response = __query_contribs(settings, ids, ohsome_filter, endpoint, time) + id_types = [(x[1], x[2]) for x in lst] + ref_ids = [x[0] for x in lst] + ohsome_filter = "id:(" + ids_to_filter(id_types) + ")" + response = __query_contribs( + settings, ref_ids, ohsome_filter, endpoint, time + ) contribs_list = __process_contrib_result( response, settings.processing.timestamp ) @@ -64,7 +66,9 @@ def get_contribs(settings: ActivatedSettings): __set_changeset_uid(settings.db_parameters) -def __upload_contribs(db_parameters: ActivatedDBConnectionManager, contrib_list: list): +def __upload_contribs( + db_parameters: ActivatedDBConnectionManager, contrib_list: List[dict] +): """Commit contributions to DB. :param contrib_list: @@ -82,7 +86,7 @@ def __upload_contribs(db_parameters: ActivatedDBConnectionManager, contrib_list: db_parameters.target_db.putconn(database) -def __set_changeset_uid(db_parameters: ActivatedDBConnectionManager): +def __set_changeset_uid(db_parameters: ActivatedDBConnectionManager) -> None: """ Set User IDs to Edits Table. @@ -116,12 +120,16 @@ def __set_changeset_uid(db_parameters: ActivatedDBConnectionManager): def __query_contribs( - settings: ActivatedSettings, lst: list, ohsome_filter: str, endpoint: str, time: str -): + settings: ActivatedSettings, + ref_id_list: List[int], + ohsome_filter: str, + endpoint: str, + time: str, +) -> List[dict]: """Query ohsome for element contributions. :param settings: - :param lst: + :param osm_id_list: :param ohsome_filter: :param endpoint: :param time: @@ -129,7 +137,7 @@ def __query_contribs( """ if settings.processing.fast_contribs: response = settings.ohsome.splitable_request( - aoi=settings.db_parameters.get_buffer_df(osm_id_list=lst), + aoi=settings.db_parameters.get_buffer_df(ref_id_list=ref_id_list), ohsome_filter=ohsome_filter, endpoint=endpoint, timestamp=time, @@ -144,7 +152,7 @@ def __query_contribs( return response -def __process_contrib_result(response: list, target_timestamp: datetime): +def __process_contrib_result(response: List[dict], target_timestamp: datetime): """Process the result of the contribution extraction. :param target_timestamp: @@ -156,13 +164,14 @@ def __process_contrib_result(response: list, target_timestamp: datetime): for res in response: for feature in res["features"]: props = feature["properties"] - ident = processing.ids_from_ohsome(props)[0] + ident, osm_type = processing.ids_from_ohsome(props) timestamp = props["@timestamp"] cs_id = props["@contributionChangesetId"] for contrib_type in types: if contrib_type in props: orig_list.append( { + "osm_type": osm_type, "osm_id": ident, "editts": timestamp, "csId": cs_id, diff --git a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json index a5d6f65..4ca5552 100644 --- a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json +++ b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json @@ -1,6 +1,6 @@ { "ref_id": "int64", - "type": "str", + "osm_type": "str", "primtag": "str", "corine_class": "str", "invalidity": "int64", diff --git a/src/osm_element_vectorisation/resources/sql/data_extraction/check_presence.sql b/src/osm_element_vectorisation/resources/sql/data_extraction/check_presence.sql index a544d8f..0fec1e3 100644 --- a/src/osm_element_vectorisation/resources/sql/data_extraction/check_presence.sql +++ b/src/osm_element_vectorisation/resources/sql/data_extraction/check_presence.sql @@ -1,27 +1,19 @@ -WITH -unnested_ids as( SELECT -unnest(ARRAY[split_part(unnest(array[%(searchIds)s]), '/', 1)]) as type, -unnest(ARRAY[split_part(unnest(array[%(searchIds)s]), '/', 2)]) as id -), -ids AS ( - SELECT - unnested_ids.id, - unnested_ids.type, - ARRAY_AGG(tstamp_download) AS tss - FROM - unnested_ids - LEFT JOIN {data_table_name} td ON - unnested_ids.id::bigint = td.osm_id AND - unnested_ids.type = td.type - GROUP BY - unnested_ids.id, - unnested_ids.type - ) -SELECT - id, type + osm_type, + osm_id FROM - ids +(VALUES %s) +AS ct( + osm_type, + osm_id + ) WHERE - NOT (%(ts)s = ANY(tss)) OR - ((%(ts)s = ANY(tss)) IS NULL); --return ids if either the ts is not present yet or the object is not present at all (tss contains only null values) + NOT EXISTS ( + SELECT + osm_id + FROM + {data_table_name} dtn + WHERE + dtn.osm_type = ct.osm_type + AND dtn.osm_id = ct.osm_id + ); diff --git a/src/osm_element_vectorisation/resources/sql/data_extraction/create_surrounding_table.sql b/src/osm_element_vectorisation/resources/sql/data_extraction/create_surrounding_table.sql index 8e0c001..0a1027d 100644 --- a/src/osm_element_vectorisation/resources/sql/data_extraction/create_surrounding_table.sql +++ b/src/osm_element_vectorisation/resources/sql/data_extraction/create_surrounding_table.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS {surrounding_table}( id serial PRIMARY KEY, + osm_type TEXT, osm_id bigint, - type TEXT, primkey TEXT, primval TEXT, othertags hstore, @@ -17,10 +17,15 @@ CREATE TABLE IF NOT EXISTS {surrounding_table}( ref_aoi integer REFERENCES {bbox_table}(id), note TEXT, UNIQUE ( + osm_type, osm_id, - tstamp_download, primkey, primval + ), + -- ensure only one timestamp is stored in the table + EXCLUDE + USING gist( + tstamp_download WITH <> ) ); diff --git a/src/osm_element_vectorisation/resources/sql/data_extraction/upload_query.sql b/src/osm_element_vectorisation/resources/sql/data_extraction/upload_query.sql index 41a0698..6bfd68f 100644 --- a/src/osm_element_vectorisation/resources/sql/data_extraction/upload_query.sql +++ b/src/osm_element_vectorisation/resources/sql/data_extraction/upload_query.sql @@ -2,7 +2,7 @@ INSERT INTO {0}( osm_id, - type, + osm_type, primkey, primval, othertags, @@ -15,7 +15,7 @@ INSERT VALUES %s ON CONFLICT ( osm_id, - tstamp_download, + osm_type, primkey, primval ) DO NOTHING RETURNING id; diff --git a/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql b/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql index 2bc9dbf..84cdfa1 100644 --- a/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql +++ b/src/osm_element_vectorisation/resources/sql/export/get_geodata.sql @@ -1,7 +1,7 @@ SELECT id, + osm_type, osm_id, - type, geom, note FROM diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/contibs_upload.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/contibs_upload.sql index 768281b..20eaad5 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/contibs_upload.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/contibs_upload.sql @@ -1,11 +1,11 @@ INSERT -INTO + INTO {edits_table_name}( changesetid, ref_id, edit_type, edit_ts - ) + ) SELECT %(csId)s, id, @@ -15,10 +15,11 @@ FROM {data_table_name} WHERE osm_id =%(osm_id)s -AND tstamp_download =%(time)s ON -CONFLICT ( - ref_id, - changesetid, - edit_ts, - edit_type + AND osm_type =%(osm_type)s + AND tstamp_download =%(time)s ON + CONFLICT ( + ref_id, + changesetid, + edit_ts, + edit_type ) DO NOTHING; diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql index 87f54cc..cae9a03 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/get_ids.sql @@ -1,5 +1,7 @@ SELECT - concat(type, '/', osm_id) + td.id, + osm_type, + osm_id FROM {data_table_name} td LEFT JOIN {edits_table_name} drie ON diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql index ebce5b1..884ad7c 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql @@ -185,7 +185,7 @@ sauce_agg AS ( ) SELECT itn.ref_id, - type, + osm_type, concat(dtn.primkey, '=', dtn.primval) AS primtag, cl.level1 AS corine_class, 3 * validity_issues_geom::integer + 2 * validity_issues_geog::integer + validity_issues_postprocess::integer + geom_change_postprocess::integer AS invalidity, diff --git a/src/osm_element_vectorisation/resources/sql/setup/check_extensions.sql b/src/osm_element_vectorisation/resources/sql/setup/check_extensions.sql index efff020..6573450 100644 --- a/src/osm_element_vectorisation/resources/sql/setup/check_extensions.sql +++ b/src/osm_element_vectorisation/resources/sql/setup/check_extensions.sql @@ -1,3 +1,4 @@ CREATE EXTENSION IF NOT EXISTS hstore; CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS postgis_raster; +CREATE EXTENSION IF NOT EXISTS btree_gist; diff --git a/src/osm_element_vectorisation/resources/sql/setup/create_required_minimum_tables.sql b/src/osm_element_vectorisation/resources/sql/setup/create_required_minimum_tables.sql index e7b22f3..a081467 100644 --- a/src/osm_element_vectorisation/resources/sql/setup/create_required_minimum_tables.sql +++ b/src/osm_element_vectorisation/resources/sql/setup/create_required_minimum_tables.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS {data_table_name} ( id serial PRIMARY KEY, + osm_type TEXT, osm_id bigint, - type TEXT, primkey TEXT, primval TEXT, othertags hstore, @@ -29,10 +29,15 @@ CREATE TABLE IF NOT EXISTS {data_table_name} ( note TEXT, ref_aoi integer REFERENCES {bbox_table}(id), UNIQUE ( + osm_type, osm_id, - tstamp_download, primkey, primval + ), + -- ensure only one timestamp is stored in the table + EXCLUDE + USING gist( + tstamp_download WITH <> ) ); -- create indices diff --git a/src/osm_element_vectorisation/resources/sql/setup/queries/check_ts.sql b/src/osm_element_vectorisation/resources/sql/setup/queries/check_ts.sql new file mode 100644 index 0000000..4a8e284 --- /dev/null +++ b/src/osm_element_vectorisation/resources/sql/setup/queries/check_ts.sql @@ -0,0 +1,7 @@ +SELECT + tstamp_download +FROM + {data_table_name} +WHERE + tstamp_download != %(ts)s +LIMIT 1; diff --git a/src/osm_element_vectorisation/resources/sql/setup/queries/get_buffer_query.sql b/src/osm_element_vectorisation/resources/sql/setup/queries/get_buffer_query.sql index f42d4c2..701f052 100644 --- a/src/osm_element_vectorisation/resources/sql/setup/queries/get_buffer_query.sql +++ b/src/osm_element_vectorisation/resources/sql/setup/queries/get_buffer_query.sql @@ -5,7 +5,6 @@ SELECT FROM {data_table_name} WHERE - id = any(%(ref_id_list)s) OR - osm_id = any(%(osm_id_list)s) + id = any(%(ref_id_list)s) ORDER BY st_xmin(buffer),st_ymin(buffer); -- GitLab From fe02106083fef08f91a97bd51115cde8cd7aeba5 Mon Sep 17 00:00:00 2001 From: Moritz Schott Date: Tue, 12 Apr 2022 14:14:16 +0200 Subject: [PATCH 13/17] prepare addition of lines and points --- doc/indicators.md | 4 +-- .../indicators/preprocessing/preprocessing.py | 29 +++---------------- .../sql/indicators/element/geometry/area.sql | 2 +- .../preprocessing/indicator_table_setup.sql | 8 ++--- 4 files changed, 11 insertions(+), 32 deletions(-) diff --git a/doc/indicators.md b/doc/indicators.md index b0e7034..dcdbbb5 100644 --- a/doc/indicators.md +++ b/doc/indicators.md @@ -8,7 +8,7 @@ The following tables provide information on the indicators calculated. Fore more | Indicator Name | Description | Calculation | Logical Link to LULC Quality | Extrinsic Data Involved | Problems | Reference | Alternative approaches | |---|---|---|---|---|---|---|---| -| relation | Is the OSM object a way or a relation? | relation? -> true:false | Relations in OSM are more complex and need higher skill to be mapped but also are more error prone. | | | | | +| osm_type | The type of the OSM object (node, way or relation) | Extracted from the object | Relations in OSM are more complex and need higher skill to be mapped but also are more error prone. | | | | | | primtag | Primary tag of the object. | Key and value of tags contained in the [MapFeatures list](https://wiki.openstreetmap.org/wiki/Map_features) | Some tags may be more difficult to map, ambiguous, less established or less documented than others. | | | | | | corine\_class | A semantic aggregation of primary tags into the CORINE class level 2. | According to the [mapping](../src/osm_element_vectorisation/resources/sql/setup/osm_tags.sql) of the primary key to CORINE land use / land cover classes. | | | Large information overlap with *primtag*. | [CORINE class definition](https://land.copernicus.eu/user-corner/technical-library/corine-land-cover-nomenclature-guidelines/html) | | @@ -20,7 +20,7 @@ The following tables provide information on the indicators calculated. Fore more | invalidity | Gravity of geometric invalidity. | The geometry is made valid using different approaches. The count of failed approaches is recorded. A value of 0 is assigned if the geometry was valid.| Invalid geometries describe bad data. | | | | | | detail | How detailed is the object drawn? | perimeter(m)/n\_vertex as the average segment length | The more detailed an object the more effort and the higher the quality | | | [Mooney at al 2010](https://www.researchgate.net/publication/229009232_A_study_of_data_representation_of_natural_features_in_OpenStreetMap) | The sum of inner angles [https://osm.zz.de/dbview/?#landuseoverlap-bw](https://osm.zz.de/dbview/?#landuseoverlap-bw) | | complexity | How complex is the geometry?| 0.8 * vibration amplitude * vibration frequency + 0.2 * deviation from convex hull. If multiple rings are present they are summed up. Vibration aplitude and frequency describe how often and strong 'notches' occur on the geometry. The result is a unitless complexity index. | | | Only defined for simple polygons. | [Brinkhoff et al 1995](https://www.researchgate.net/publication/221589831_Measuring_the_Complexity_of_Polygonal_Objects)|[https://ieeexplore.ieee.org/abstract/document/4014089](https://ieeexplore.ieee.org/abstract/document/4014089)| -| obj\_area | Size of the object. | area(sqm) | In combination with the region or continent, where the object is located, this might show oversized elements. The trend in OSM goes towards smaller elements because they can better describe local detail → the smaller the element the better (up to a certain point). | | | |May be combined with other indicators to normalise (e.g. size in relation to mean size for this tag (is it a large farmland (in this area ?) or in relation to similar objects).| +| obj\_size | Size of the object. | area(sqm) | In combination with the region or continent, where the object is located, this might show oversized elements. The trend in OSM goes towards smaller elements because they can better describe local detail → the smaller the element the better (up to a certain point). | | | |May be combined with other indicators to normalise (e.g. size in relation to mean size for this tag (is it a large farmland (in this area ?) or in relation to similar objects).| ## Surrounding diff --git a/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py b/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py index 2285ef0..20a25f5 100644 --- a/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py +++ b/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py @@ -23,11 +23,7 @@ def preprocess(settings: ActivatedSettings): def get_contribs(settings: ActivatedSettings): - """Retrieve contributions to extracted elements. - - :param settings: - :return: - """ + """Retrieve contributions to extracted elements.""" logging.info("request contributions") # in an earlier version and before this endpoint was available # this was done using the oshdb-api @@ -69,11 +65,7 @@ def get_contribs(settings: ActivatedSettings): def __upload_contribs( db_parameters: ActivatedDBConnectionManager, contrib_list: List[dict] ): - """Commit contributions to DB. - - :param contrib_list: - :return: - """ + """Commit contributions to DB.""" logging.debug("commit contribs to postgres") # do not use other upload functions as they request new curser for each row query = sql.SQL(read_text(sql_files, "contibs_upload.sql")).format( @@ -126,15 +118,7 @@ def __query_contribs( endpoint: str, time: str, ) -> List[dict]: - """Query ohsome for element contributions. - - :param settings: - :param osm_id_list: - :param ohsome_filter: - :param endpoint: - :param time: - :return: - """ + """Query ohsome for element contributions.""" if settings.processing.fast_contribs: response = settings.ohsome.splitable_request( aoi=settings.db_parameters.get_buffer_df(ref_id_list=ref_id_list), @@ -153,12 +137,7 @@ def __query_contribs( def __process_contrib_result(response: List[dict], target_timestamp: datetime): - """Process the result of the contribution extraction. - - :param target_timestamp: - :param response: - :return: - """ + """Process the result of the contribution extraction.""" orig_list = [] types = ["@creation", "@geometryChange", "@tagChange", "@deletion"] for res in response: diff --git a/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql b/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql index 84cb624..abf435a 100644 --- a/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql +++ b/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql @@ -3,7 +3,7 @@ UPDATE {indicator_table_name} SET - obj_area = st_area(geom::geography) + obj_size = st_area(geom::geography) FROM {data_table_name} oad WHERE diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql index 884ad7c..4646a00 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql @@ -7,7 +7,7 @@ country integer REFERENCES {countries_table}(ogc_fid), hdi integer REFERENCES {hdi_table}(ogc_fid), biome integer REFERENCES {metadata_schema}.biomes_legend(biome_id), - --cultural areas LEFT out + --'cultural areas' were excluded validity_issues_geom boolean, validity_issues_geog boolean, validity_issues_postprocess boolean, @@ -19,7 +19,7 @@ conv NUMERIC, shared_border NUMERIC, perimeter NUMERIC, - obj_area NUMERIC, + obj_size NUMERIC, element_overlap NUMERIC, uptodateness INTERVAL, obj_age INTERVAL, @@ -191,14 +191,14 @@ SELECT 3 * validity_issues_geom::integer + 2 * validity_issues_geog::integer + validity_issues_postprocess::integer + geom_change_postprocess::integer AS invalidity, itn.perimeter/n_vertex AS detail, 0.8 * ampl*freq + 0.2 * conv AS complexity, - itn.obj_area, + itn.obj_size, element_overlap AS overlap, shared_border, surrounding_mapped, surrounding_diversity, ( nodes_count + ways_count - )/ itn.obj_area AS surrounding_obj_dens, --(nodes_count + ways_count + relations_count) / itn.area as surrounding_obj_dens, + )/ itn.obj_size AS surrounding_obj_dens, --(nodes_count + ways_count + relations_count) / itn.area as surrounding_obj_dens, continent AS continent, pop5k, hdi.hdi AS hdi, -- GitLab From 2d63c7be64cda017d8a4084633d934de5d53157a Mon Sep 17 00:00:00 2001 From: Moritz Schott Date: Tue, 12 Apr 2022 14:25:29 +0200 Subject: [PATCH 14/17] rename area --- .../resources/misc/predictors_dtypes.json | 2 +- .../resources/sql/indicators/element/geometry/area.sql | 2 +- vectorisation_docker.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json index 4ca5552..ef765ff 100644 --- a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json +++ b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json @@ -6,7 +6,7 @@ "invalidity": "int64", "detail": "float64", "complexity": "float64", - "obj_area": "float64", + "obj_size": "float64", "overlap": "float64", "shared_border": "float64", "surrounding_mapped": "float64", diff --git a/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql b/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql index abf435a..f1e6944 100644 --- a/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql +++ b/src/osm_element_vectorisation/resources/sql/indicators/element/geometry/area.sql @@ -8,4 +8,4 @@ FROM {data_table_name} oad WHERE ref_id = oad.id - AND obj_area IS NULL; + AND obj_size IS NULL; diff --git a/vectorisation_docker.sh b/vectorisation_docker.sh index 86bc1ed..b02795c 100755 --- a/vectorisation_docker.sh +++ b/vectorisation_docker.sh @@ -31,8 +31,8 @@ case $1 in ;; dev) docker-compose --project-directory . -f Docker/docker-compose.yaml build - #docker-compose --project-directory . -f Docker/docker-compose.yaml up postgis docker-compose --project-directory . -f Docker/docker-compose.yaml run import repex + #docker-compose --project-directory . -f Docker/docker-compose.yaml run --entrypoint bash vectorisation # download by id -- GitLab From a13cdf939e069794ce1605178a60f674a51a4303 Mon Sep 17 00:00:00 2001 From: Moritz Schott Date: Tue, 12 Apr 2022 15:03:52 +0200 Subject: [PATCH 15/17] rename indicators --- doc/indicators.md | 4 +-- .../config/db_parameters.py | 2 +- .../config/processing.py | 12 ++------ .../data_extraction/data_extraction.py | 7 +---- .../indicators/user/calc_user.py | 30 +++++++++---------- .../resources/misc/predictors_dtypes.json | 4 +-- .../sql/classify/setup_classification.sql | 4 +-- .../sql/classify/upload_indicators_norm.sql | 8 ++--- .../upload_indicators_norm_template.sql | 4 +-- .../sql/data_extraction/insert_ref.sql | 3 +- .../sql/data_extraction/target_check.sql | 8 ----- .../resources/sql/indicators/element/time.sql | 6 ++-- .../{calcLocalness.sql => calcRemoteness.sql} | 4 +-- .../{get_localness.sql => get_remoteness.sql} | 0 ...calness.sql => get_user_id_remoteness.sql} | 2 +- ...te_localness.sql => update_remoteness.sql} | 2 +- .../preprocessing/indicator_table_setup.sql | 10 +++---- 17 files changed, 45 insertions(+), 65 deletions(-) delete mode 100644 src/osm_element_vectorisation/resources/sql/data_extraction/target_check.sql rename src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/{calcLocalness.sql => calcRemoteness.sql} (96%) rename src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/{get_localness.sql => get_remoteness.sql} (100%) rename src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/{get_user_id_localness.sql => get_user_id_remoteness.sql} (94%) rename src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/{update_localness.sql => update_remoteness.sql} (84%) diff --git a/doc/indicators.md b/doc/indicators.md index dcdbbb5..faa9a47 100644 --- a/doc/indicators.md +++ b/doc/indicators.md @@ -50,7 +50,7 @@ The following tables provide information on the indicators calculated. Fore more | Indicator Name | Description | Calculation | Logical Link to LULC Quality | Extrinsic Data Involved | Problems | Reference | Alternative approaches | |---|---|---|---|---|---|---|---| -| uptodateness |How up to date is the object| time in seconds since last minor OSHDB version (user touch of element or member) | Recently changed objects are potentially more up to date. | | Only because I changed one edge of a large relation does not mean I checked the whole object. | [https://is-osm-uptodate.frafra.eu](https://is-osm-uptodate.frafra.eu)| | +| outofdateness |How outdated is the object| time in seconds since last minor OSHDB version (user touch of element or member) | Recently changed objects are potentially more up to date. | | Only because I changed one edge of a large relation does not mean I checked the whole object. If the object is perfectly drawn and stable it may not change for a long time. | [https://is-osm-uptodate.frafra.eu](https://is-osm-uptodate.frafra.eu)| | | object\_age | How old is the object? | time in seconds since creation of object | Old objects are either outdated and were mapped with superseded mapping practices or had much time to mature with many small improvements. | | | | | | changes\_per\_year | How frequently has the object been changed/updated | n\_changes/object\_age | Objects that are regularly changed are well maintained. | | Not relevant for new objects | | | @@ -60,7 +60,7 @@ The following tables provide information on the indicators calculated. Fore more | Indicator Name | Description | Calculation | Logical Link to LULC Quality | Extrinsic Data Involved | Problems | Reference | Alternative approaches | |---|---|---|---|---|---|---|---| | user\_mean\_exp | How experienced were the users that changed this object| weighted mean number of changesets of users that edited the object | More experience users may have a higher probability to make changes that improve quality. | | |**Add Schott et al and sources therin (Neis)**| Experience is not well defined. Alternatives are: time since first edit; number of edits... . Weighting is arbitrary. Most objects are at version 1 so creation may be most important. Others have created complex experience indicators: [https://onlinelibrary.wiley.com/doi/full/10.1111/tgis.12454](https://onlinelibrary.wiley.com/doi/full/10.1111/tgis.12454)| -| user\_localness | How local where the mapper(s) of the object? | median distance of changeset centroids to element centroid | Local users are more familiar with the region and should create better data. | [Changesets](https://wiki.openstreetmap.org/wiki/Changeset)| HOT users will be located in Africa; only the localness of the most recent editor of an objects is calculated. | **add Neis** | Alternatives described in the reference| +| user\_remoteness | How far away where other edits done by the mapper(s) of the object? | median distance of changeset centroids to element centroid | Local users are more familiar with the region and should create better data. | [Changesets](https://wiki.openstreetmap.org/wiki/Changeset)| HOT users will be located in Africa; only the remoteness of the most recent editor of an objects is calculated. | **add Neis** | Alternatives described in the reference| | user\_diversity | How specialised is the user|[shannon index](https://de.wikipedia.org/wiki/Shannon-Index) if edits made by user grouped by [tagGrouping.json](../src/osm_element_vectorisation/resources/misc/tagGrouping.json)| Diverse users may be general experts but specialized users may be experts on this topic.| | | **add Schott et al**| An alternative would be the average location or the median distance to the corners of the changesets. This would penalize more for large changesets which are relatively rare but a sign of bad quality.| ## Mapping process diff --git a/src/osm_element_vectorisation/config/db_parameters.py b/src/osm_element_vectorisation/config/db_parameters.py index 530e2a2..363034b 100644 --- a/src/osm_element_vectorisation/config/db_parameters.py +++ b/src/osm_element_vectorisation/config/db_parameters.py @@ -118,7 +118,7 @@ class DBConnectionManager: return res def get_buffer_df(self, ref_id_list: List[int]) -> geopandas.GeoDataFrame: - """Get GDF of buffer geoms with given ref_id or osm_id.""" + """Get GDF of buffer geoms with given ref_id.""" buf = self.get_db_geopandas( query=read_text(queries, "get_buffer_query.sql"), params={"ref_id_list": ref_id_list}, diff --git a/src/osm_element_vectorisation/config/processing.py b/src/osm_element_vectorisation/config/processing.py index 6314165..1cf0cf7 100644 --- a/src/osm_element_vectorisation/config/processing.py +++ b/src/osm_element_vectorisation/config/processing.py @@ -81,11 +81,7 @@ class Processing: def ids_from_ohsome(props: dict) -> Tuple[int, str]: - """Convert ohsome props to id and object type. - - :param props: - :return: - """ + """Convert ohsome props to id and object type.""" if "node" in props["@osmId"]: ident = int(re.sub("node/", "", props["@osmId"])) obj_type = "node" @@ -103,10 +99,6 @@ def ids_from_ohsome(props: dict) -> Tuple[int, str]: def ids_to_filter(id_list: List[Tuple[str, int]]) -> str: - """Convert positive and negative ids to way and relation filter. - - :param id_list: - :return: - """ + """Convert positive and negative ids to way and relation filter.""" osm_ids = ",".join([x[0] + "/" + str(x[1]) for x in id_list]) return osm_ids diff --git a/src/osm_element_vectorisation/data_extraction/data_extraction.py b/src/osm_element_vectorisation/data_extraction/data_extraction.py index 8266058..1fd7b00 100644 --- a/src/osm_element_vectorisation/data_extraction/data_extraction.py +++ b/src/osm_element_vectorisation/data_extraction/data_extraction.py @@ -103,12 +103,7 @@ def download_by_bbox(settings: ActivatedSettings) -> List[int]: def __get_download_filter(osm_ids: str, tags: str) -> str: - """Place givens in an ohsome filter. - - :param osm_ids: - :param tags: - :return: - """ + """Place givens in an ohsome filter.""" ohsome_filter = f"({tags}) and geometry:polygon " f"and (id:({osm_ids}))" return ohsome_filter diff --git a/src/osm_element_vectorisation/indicators/user/calc_user.py b/src/osm_element_vectorisation/indicators/user/calc_user.py index 60482df..95f6cec 100644 --- a/src/osm_element_vectorisation/indicators/user/calc_user.py +++ b/src/osm_element_vectorisation/indicators/user/calc_user.py @@ -20,7 +20,7 @@ def calc_user(settings: ActivatedSettings): if settings.db_parameters.changesets_db: calc_experience(settings) - calc_localness(settings) + calc_remoteness(settings) if settings.oshdb: calc_diversity(settings) @@ -31,7 +31,7 @@ def calc_user(settings: ActivatedSettings): else: logging.warning( "changesets database connection not provided, " - "skipping user experience, localness and diversity" + "skipping user experience, remoteness and diversity" ) @@ -73,39 +73,39 @@ def __upload_user_experience( ) -def calc_localness(settings: ActivatedSettings): - """Calculate localness of users.""" - logging.info("calculating localness") +def calc_remoteness(settings: ActivatedSettings): + """Calculate remoteness of users.""" + logging.info("calculating user remoteness") if settings.db_parameters.changesets_db == settings.db_parameters.target_db: settings.db_parameters.ex_iud( - read_text(changeset_based, "calcLocalness.sql"), + read_text(changeset_based, "calcRemoteness.sql"), variables={"ts": settings.processing.timestamp}, ) else: uids = settings.db_parameters.ex_select( - read_text(changeset_based, "get_user_id_localness.sql") + read_text(changeset_based, "get_user_id_remoteness.sql") ) if uids: - user_localness_mapping = __get_localness(settings, uids) - __upload_localness(settings.db_parameters, user_localness_mapping) + user_remoteness_mapping = __get_remoteness(settings, uids) + __upload_remoteness(settings.db_parameters, user_remoteness_mapping) -def __get_localness(settings: ActivatedSettings, uids: list): +def __get_remoteness(settings: ActivatedSettings, uids: list): return settings.db_parameters.ex_select_cs( - sql_string=read_text(changeset_based, "get_localness.sql"), + sql_string=read_text(changeset_based, "get_remoteness.sql"), variables={"vals": uids, "ts": settings.processing.timestamp}, ) -def __upload_localness( - db_parameters: ActivatedDBConnectionManager, user_localness_mapping: list +def __upload_remoteness( + db_parameters: ActivatedDBConnectionManager, user_remoteness_mapping: list ): db_parameters.ex_sql_batch_upload( - query=read_text(changeset_based, "update_localness.sql"), + query=read_text(changeset_based, "update_remoteness.sql"), template="(%s, %s)", - value_list=user_localness_mapping, + value_list=user_remoteness_mapping, fetch=False, ) diff --git a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json index ef765ff..5e776ff 100644 --- a/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json +++ b/src/osm_element_vectorisation/resources/misc/predictors_dtypes.json @@ -16,7 +16,7 @@ "pop5k": "float64", "hdi": "float64", "biome": "str", - "uptodateness": "float64", + "outofdateness": "float64", "object_age": "float64", "changes_per_year": "float64", "user_mean_exp": "float64", @@ -24,7 +24,7 @@ "cs_median_area": "float64", "cs_editor": "str", "how_many_eyes": "float64", - "user_localness": "float64", + "user_remoteness": "float64", "user_diversity": "float64", "validator_issues_density": "float64", "notes_density": "float64", diff --git a/src/osm_element_vectorisation/resources/sql/classify/setup_classification.sql b/src/osm_element_vectorisation/resources/sql/classify/setup_classification.sql index 3d81d9e..20bd811 100644 --- a/src/osm_element_vectorisation/resources/sql/classify/setup_classification.sql +++ b/src/osm_element_vectorisation/resources/sql/classify/setup_classification.sql @@ -33,13 +33,13 @@ CREATE TABLE IF NOT EXISTS {indicators_norm_table_name} ( surrounding_obj_dens NUMERIC, pop5k NUMERIC, hdi NUMERIC, - uptodateness NUMERIC, + outofdateness NUMERIC, object_age NUMERIC, changes_per_year NUMERIC, user_mean_exp NUMERIC, cs_median_area NUMERIC, how_many_eyes NUMERIC, - user_localness NUMERIC, + user_remoteness NUMERIC, user_diversity NUMERIC, validator_issues_density NUMERIC, notes_density NUMERIC, diff --git a/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm.sql b/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm.sql index 06a284d..7e1139d 100644 --- a/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm.sql +++ b/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm.sql @@ -12,13 +12,13 @@ SET surrounding_obj_dens = data.surrounding_obj_dens, pop5k = data.pop5k, hdi = data.hdi, - uptodateness = data.uptodateness, + outofdateness = data.outofdateness, object_age = data.object_age, changes_per_year = data.changes_per_year, user_mean_exp = data.user_mean_exp, cs_median_area = data.cs_median_area, how_many_eyes = data.how_many_eyes, - user_localness = data.user_localness, + user_remoteness = data.user_remoteness, user_diversity = data.user_diversity, validator_issues_density = data.validator_issues_density, notes_density = data.notes_density, @@ -79,13 +79,13 @@ FROM surrounding_obj_dens, pop5k, hdi, - uptodateness, + outofdateness, object_age, changes_per_year, user_mean_exp, cs_median_area, how_many_eyes, - user_localness, + user_remoteness, user_diversity, validator_issues_density, notes_density, diff --git a/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm_template.sql b/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm_template.sql index fabda9a..7d8d3e4 100644 --- a/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm_template.sql +++ b/src/osm_element_vectorisation/resources/sql/classify/upload_indicators_norm_template.sql @@ -11,13 +11,13 @@ %(surrounding_obj_dens)s, %(pop5k)s, %(hdi)s, - %(uptodateness)s, + %(outofdateness)s, %(object_age)s, %(changes_per_year)s, %(user_mean_exp)s, %(cs_median_area)s, %(how_many_eyes)s, - %(user_localness)s, + %(user_remoteness)s, %(user_diversity)s, %(validator_issues_density)s, %(notes_density)s, diff --git a/src/osm_element_vectorisation/resources/sql/data_extraction/insert_ref.sql b/src/osm_element_vectorisation/resources/sql/data_extraction/insert_ref.sql index 16dc2ca..1a3eb1c 100644 --- a/src/osm_element_vectorisation/resources/sql/data_extraction/insert_ref.sql +++ b/src/osm_element_vectorisation/resources/sql/data_extraction/insert_ref.sql @@ -1,5 +1,5 @@ --relate objects to surrounding --- filter objects already are related +-- filter objects that are already related WITH missing_ids AS ( SELECT id @@ -26,6 +26,7 @@ LEFT JOIN {surrounding_table} sd ON ) AND od.tstamp_download = %(time)s AND od.osm_id != sd.osm_id +AND od.osm_type != sd.osm_type ) INSERT INTO diff --git a/src/osm_element_vectorisation/resources/sql/data_extraction/target_check.sql b/src/osm_element_vectorisation/resources/sql/data_extraction/target_check.sql deleted file mode 100644 index 53e9409..0000000 --- a/src/osm_element_vectorisation/resources/sql/data_extraction/target_check.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - id, - osm_id -FROM - {data_table_name} -WHERE - ref_aoi=%(aoi)s -LIMIT 1; diff --git a/src/osm_element_vectorisation/resources/sql/indicators/element/time.sql b/src/osm_element_vectorisation/resources/sql/indicators/element/time.sql index 934d203..36aac8d 100644 --- a/src/osm_element_vectorisation/resources/sql/indicators/element/time.sql +++ b/src/osm_element_vectorisation/resources/sql/indicators/element/time.sql @@ -1,5 +1,5 @@ ---uptodateness +--outofdateness WITH ts AS ( SELECT od.id, @@ -14,12 +14,12 @@ UPDATE {indicator_table_name} SET - uptodateness = lastDur + outofdateness = lastDur FROM ts WHERE ts.id = ref_id - AND uptodateness IS NULL; + AND outofdateness IS NULL; --age WITH ts AS ( diff --git a/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/calcLocalness.sql b/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/calcRemoteness.sql similarity index 96% rename from src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/calcLocalness.sql rename to src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/calcRemoteness.sql index 1428d7a..a5ac4f9 100644 --- a/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/calcLocalness.sql +++ b/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/calcRemoteness.sql @@ -7,7 +7,7 @@ WITH ts AS ( -- GET latest edit timestamp OF an object JOIN {indicator_table_name} it ON drie.ref_id = it.ref_id WHERE - it.user_localness IS NULL + it.user_remoteness IS NULL GROUP BY drie.ref_id ), @@ -54,7 +54,7 @@ perc AS ( ) UPDATE {indicator_table_name} AS itn SET - user_localness = dist + user_remoteness = dist FROM perc WHERE diff --git a/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_localness.sql b/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_remoteness.sql similarity index 100% rename from src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_localness.sql rename to src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_remoteness.sql diff --git a/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_user_id_localness.sql b/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_user_id_remoteness.sql similarity index 94% rename from src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_user_id_localness.sql rename to src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_user_id_remoteness.sql index c9b4e47..cf7c5ee 100644 --- a/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_user_id_localness.sql +++ b/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/get_user_id_remoteness.sql @@ -7,7 +7,7 @@ WITH ts AS ( -- GET latest edit timestamp OF an object JOIN {indicator_table_name} it ON drie.ref_id = it.ref_id WHERE - it.user_localness IS NULL + it.user_remoteness IS NULL GROUP BY drie.ref_id ) diff --git a/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/update_localness.sql b/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/update_remoteness.sql similarity index 84% rename from src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/update_localness.sql rename to src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/update_remoteness.sql index 3784610..a6a65b9 100644 --- a/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/update_localness.sql +++ b/src/osm_element_vectorisation/resources/sql/indicators/user/changeset_based/update_remoteness.sql @@ -1,7 +1,7 @@ UPDATE {indicator_table_name} AS itn SET - user_localness = dist + user_remoteness = dist FROM ( VALUES %s diff --git a/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql b/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql index 4646a00..f1b5fe6 100644 --- a/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql +++ b/src/osm_element_vectorisation/resources/sql/preprocessing/indicator_table_setup.sql @@ -21,7 +21,7 @@ perimeter NUMERIC, obj_size NUMERIC, element_overlap NUMERIC, - uptodateness INTERVAL, + outofdateness INTERVAL, obj_age INTERVAL, surrounding_mapped NUMERIC, surrounding_diversity integer, @@ -36,7 +36,7 @@ user_mean_exp NUMERIC, user_creation_diversity NUMERIC, avg_uniq_user_count NUMERIC, - user_localness NUMERIC, + user_remoteness NUMERIC, cs_editor TEXT[], cs_median_area NUMERIC, cs_source TEXT[], @@ -206,8 +206,8 @@ SELECT EXTRACT( epoch FROM - uptodateness - ) AS uptodateness, + outofdateness + ) AS outofdateness, EXTRACT( epoch FROM @@ -230,7 +230,7 @@ SELECT ) , COALESCE(avg_uniq_user_count, 0) -- 0 FOR elements created AFTER the LAST time-RANGE that still have NULL ) AS how_many_eyes, - user_localness, + user_remoteness, user_creation_diversity AS user_diversity, 1.0 *( n_osmose_err + n_keepright_err -- GitLab From 08c7601e186e5061570691f062d17ab9c2225475 Mon Sep 17 00:00:00 2001 From: Moritz Schott Date: Tue, 12 Apr 2022 16:34:10 +0200 Subject: [PATCH 16/17] update r models --- doc/workflow.md | 8 +++--- .../config/config.py | 2 +- .../config/processing.py | 24 ++++-------------- .../data_extraction/data_extraction.py | 2 +- .../indicators/preprocessing/preprocessing.py | 2 +- .../resources/misc/r/Rf_sem_dist.rds | Bin 415241 -> 412718 bytes .../resources/misc/r/qRf_geom_err.rds | Bin 1887211 -> 1886174 bytes .../resources/misc/r/qRf_overall.rds | Bin 1873871 -> 1870903 bytes .../misc/r/trainedPreprocessingRecipe.rds | Bin 2508385 -> 2515736 bytes .../sql/classify/setup_classification.sql | 9 ++++--- .../sql/classify/upload_indicators_norm.sql | 10 +++++--- .../upload_indicators_norm_template.sql | 5 ++-- .../sql/classify/upload_predictions.sql | 4 +-- 13 files changed, 28 insertions(+), 38 deletions(-) diff --git a/doc/workflow.md b/doc/workflow.md index 9bb44c2..1105373 100644 --- a/doc/workflow.md +++ b/doc/workflow.md @@ -14,11 +14,11 @@ Calculation of the intrinsic or semi-intrinsic quality [indicators](indicators.m ## Quality Estimation/Classification This step is experimental. Three models have been trained that will predict the overall quality of the extracted elements as well as their geometric and semantic quality. This procedure uses the [R scripting language](https://www.r-project.org/). - - For the semantic quality the model predicts the _probability_ for the primary tag to be correct using a [random forest](https://cran.r-project.org/web/packages/randomForest/randomForest.pdf) classifier. - - For the geometric quality the model predicts the error of commission, meaning the relative areal over estimation. It uses a [quantile forest regression](https://cran.r-project.org/web/packages/quantregForest/quantregForest.pdf) using the median prediction. - - The overall quality of the object is equally predicted via a quantile forest regression. The overall quality is defined as `if (primary tag is correct) {geometrc quality} else {0}`. In addition to the median prediction, the other quartiles are also reported. + - For the semantic dimension the model predicts the _probability_ for the primary tag to be correct using a [random forest](https://cran.r-project.org/web/packages/randomForest/randomForest.pdf) classifier. + - For the geometric dimension the model predicts the error of commission, meaning the relative areal over-estimation. It uses a [quantile forest regression](https://cran.r-project.org/web/packages/quantregForest/quantregForest.pdf) using the median prediction. + - The overall quality of the object is equally predicted via a quantile forest regression. The overall quality is defined as `if (primary tag is correct) {1-comission} else {0}`. In addition to the median prediction, the other quartiles are also reported. -During quality prediction indicators are normalised automatically. +Model performance and analyses will be published in a future release. During quality prediction indicators are normalised automatically. ## Resulting Data The tool populates the provided database and is resilient against multiple runs (it will ignore/reuse any existing data). Indicators are available in the table `indicators_view` in your defined data schema of the target database. It is linked to your specified data table via the column `ref_id`. The data can also be exported to files for easier access. Namely the geodata, the raw indicators, the normalised indicators and the prediction results are exported. diff --git a/src/osm_element_vectorisation/config/config.py b/src/osm_element_vectorisation/config/config.py index a0589bc..2b1d1c7 100644 --- a/src/osm_element_vectorisation/config/config.py +++ b/src/osm_element_vectorisation/config/config.py @@ -150,7 +150,7 @@ def __get_limit_ids(limit_ids) -> Optional[List[Tuple[str, int]]]: logging.error("did not understand input for ids") sys.exit(1) - id_list = [(x.split("/")[0], int(x.split("/")[1])) for x in id_list] + id_list = [(x.split("/")[0].lower(), int(x.split("/")[1])) for x in id_list] return id_list diff --git a/src/osm_element_vectorisation/config/processing.py b/src/osm_element_vectorisation/config/processing.py index 1cf0cf7..029b7e2 100644 --- a/src/osm_element_vectorisation/config/processing.py +++ b/src/osm_element_vectorisation/config/processing.py @@ -1,10 +1,8 @@ """Class for processing settings.""" # pylint: disable=too-many-arguments # pylint: disable=too-many-instance-attributes -import logging import os import pathlib -import re from datetime import datetime, timedelta from logging import INFO from typing import List, Tuple @@ -80,25 +78,13 @@ class Processing: return time_range -def ids_from_ohsome(props: dict) -> Tuple[int, str]: - """Convert ohsome props to id and object type.""" - if "node" in props["@osmId"]: - ident = int(re.sub("node/", "", props["@osmId"])) - obj_type = "node" - elif "way" in props["@osmId"]: - ident = int(re.sub("way/", "", props["@osmId"])) - obj_type = "way" - elif "relation" in props["@osmId"]: - ident = int(re.sub("relation/", "", props["@osmId"])) - obj_type = "relation" - else: - logging.error("Unknown osm type in response. The response may be corrupt.") - return -1, "unknown" - - return ident, obj_type +def ids_from_ohsome(ohsome_id: str) -> Tuple[str, int]: + """Convert ohsome id to id and object type.""" + split = ohsome_id.split("/") + return split[0], int(split[1]) def ids_to_filter(id_list: List[Tuple[str, int]]) -> str: - """Convert positive and negative ids to way and relation filter.""" + """Convert osm ids and type to ohsome filter.""" osm_ids = ",".join([x[0] + "/" + str(x[1]) for x in id_list]) return osm_ids diff --git a/src/osm_element_vectorisation/data_extraction/data_extraction.py b/src/osm_element_vectorisation/data_extraction/data_extraction.py index 1fd7b00..dbd12c9 100644 --- a/src/osm_element_vectorisation/data_extraction/data_extraction.py +++ b/src/osm_element_vectorisation/data_extraction/data_extraction.py @@ -149,7 +149,7 @@ def __parse_response_list( for response in response_list: for feature in response["features"]: props = feature["properties"] - ident, obj_type = processing.ids_from_ohsome(props) + obj_type, ident = processing.ids_from_ohsome(props["@osmId"]) geom = json.dumps(feature["geometry"]) # for each property, check if it is a primary tag as defined by the user diff --git a/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py b/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py index 20a25f5..c97ec46 100644 --- a/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py +++ b/src/osm_element_vectorisation/indicators/preprocessing/preprocessing.py @@ -143,7 +143,7 @@ def __process_contrib_result(response: List[dict], target_timestamp: datetime): for res in response: for feature in res["features"]: props = feature["properties"] - ident, osm_type = processing.ids_from_ohsome(props) + osm_type, ident = processing.ids_from_ohsome(props["@osmId"]) timestamp = props["@timestamp"] cs_id = props["@contributionChangesetId"] for contrib_type in types: diff --git a/src/osm_element_vectorisation/resources/misc/r/Rf_sem_dist.rds b/src/osm_element_vectorisation/resources/misc/r/Rf_sem_dist.rds index bb3fd95cf1ba12f6632f5ce358ea5ca62334c591..4100676b04a1155cd8358372f48af56db3694ee8 100644 GIT binary patch literal 412718 zcmeFZdstJ~w(wm`TXsdEt!=4tNn3ZN7KJL6TOg_1ZP``*3Tlwt4yYNh4ve)K*k)E|&#lv`Z7ZDmpG{{lQ;l{ZqNu-?);vKw zn?$p3r`cDehEVq-n7u>UHWNRQ|Luu|rL$b`+=W)FPb9PhwL~`A z3#;8~njz+&zzlOx0}^B?2lkG3V*##YPS>Yk;j0zMVs1Q zZkGZ$dHJH9pjqbF*(N)O&B(MG4zNu>BgH!DU8|4x)@%Ygp+eCzBYQiI9@P`2wr1#| zEJlJ|&U+wSB)Jznvh^Zg!(=_teYWT(#u>qQ9}ci1fbl^Q4*&z$v3o zo<)6u>RBc|nt3CH<e5$dUO9vHe_($K<@PbujxGz|2g~H zR<9d-BEB2k_bT_lWVa3O>(Biyz-xF<#DC6SewF**zE^lT5hKca#HnNbvaA*8;?}t5 z;;r)sPcr^sKCWL&+g43a9U8eTy0b4XAU)$3ME`#jhNbhW>Hki=pYD5?{_n)M>Av^r z|4w|C?t72^@5I)0-)j256D!ke@6!LB_#wUaKK*J;e z=ML@v$EJv}h9 zua-b9vlZ}U9Nlu0z6s#iMkC>tT3oK8M16ulC@{awbGPNxlz^yO6l7v%Ct1=x07gg& z)&&f@SO zwWW}^kY;NHVOjSmK-p10VkU#>l2;JRxVjY9VTB3x_s+(e<$N}}9`~-pm=H=wQLz?&BM|PMmM>m(v)u2AX?PFa74OM+bxKDnP{fT*39c;4f)g{FF zVDwcG*loY+y7FBEXg~DY)UUROqA!5A+RKwi>N@ttXXLu5HkLkXI_7p`2Qau<74$Z{ zE{ujhEDpS5n4i09!bE3(|HwZN0J2LaVyj!%W%%DPe9o`_*5@kUVt z){M5>%#QQMsLTGe1oX77V)Jc#-tE{P)A_yLHwNDs-^9t`NV3mW#Hn~@1|i){3{)PP zeW_C7%Cq3P&lF&hnXR*|55hLNAIz}SqH~Ihe4Nz zxp2r70weS2@)jm{vh^aLEEek7kV;iTUb}QVoS^f-bXCI9m0C-zDo?F8&rymQA_W?g ztqYgJ-oOk@H~Z4~rx>8tb=|Vf3CyR)A}ywK;i)RJmwlGLI$COZ*x!6=6lRoC>IiyR zjgY6uV$SD`00NjP3cpk}U7`YA7(-d?rJUz5TzIr0AGCUZ3 zp>VWnV!BFOQ%U^^93{4m0W0E;xE4n1vd^9>+;oFHsd8>RxeD9(aQfsP9%0h@NSEs? z+-dyTTkIWec{H7ixh`-5=5N+yJw*MMH5WKy+Y9m^?U}=5cu;GYHoqr`bo=pK<^xOQ z4G(Sg8=Ef)-Grzv{5wlLT}D$kh@TYh*BO3_vNNpq{O1JOhq^&35u*ZuS}q{T{q!CS(+j>7E^tgjvF#UtpY7&R4?{kCpNvz1)3(Z%cQ3! z!p0G7O3eO-)pQkY!~j1)SE0c@#mxvV)YrM{M;(QJLu2N4D0V+Ryx8Bki+_5}2Cu!qsx2fA zJajxE_MgzyeaWXmHc#6Yxc=eA(_g2`NenG>%&qjfOqaTsw@d92u^GcQX;2TI0H%}= zVvzF-_WLCyyQJq+`|8Zzwbs~Z#j4b=AQE_+R=$^UD>(0|*%*_y{Or!e*;Ju^Amw5; z!W``ve)PPd2N=m|7_(OT5)$3LXuHgfmx#hr9mIaanY_$I@NkBlYMUhzK)r#JMJ6p# z6vRKA{nqMYI{{|nl@0C?-1&suu@rkiM&ArwH!XzYmhKeLtwZ(5uqm4t!n}*qk)%`2 zi0HMR2?#lXFiFWE(DQmri{~5St6t%>qHH-ntq8~j+>zKP`n0&RwuQx=AkPeGWdS!i z4X+(UO?<~+ZJRt@A?gPDt+rl8bYs1nilG^<5FZDR$=tY{W77Og1tE;CW1`-~ahNIf zAHx~NJSlVT491`4;!K8XGj}q_oo}3Xmgd(bDAzQ%{wQzcj$pg!>SpOJXr|5AJhzi5 zw?>th(IvuG2xX4*=C>C+ zMut828Uk#4w)rM`pA6I@xw5t0_Me5<~MWpI}8rHMHX1I78 z9hMbG$m2Z}#J4?3_{nm`-#Xp*2g2G_Nx}WJTyz`iLJu!Aq6PV5bPUH`%(QOfV9>ti zo*n2t0Kpi@9A_OsS^L`5LON~0HW$8`36z#cl!6PUs_53qj3ANv9t~j*izpjCFSwqD zo9VrTdhIHr2DVXd^aut=`*D6(2^@PCVzwj#5Xe#_E^PsA{14JurIdB$pPu^fW9xp z9=~*>gJdV6qejtY7pj(ukbjyH-p)Uv?~dG{ZlBy&BrT-|)Q=?#O&I5Y4ge0Ji9w*Oy{W?gTUgDzGYhBQEe|MT6d-Hn6pTwF8ln7EEu^T8)O0EPVW=AM0KxLxe;o#9^cPWck` z#)%AhcRD(nRty*BnCo4k!+z`xqq8%`@E}2%XuVaO3Dhzp;p(SkptKX(DlZWjUM(~y zADvaEajZCC&M|k_R5O)l;y5Q?$ZhD&=D7{;%7lY(7DFh6dNj~Aw-T&5zXPA!knq!!@~zmjc+SF;3JTa->;?o1 zT%ICDrQuPw9)2k-5wLXt$;`Soc0#`9;%a;|kDtl45m?K@2uNeGjjg%l3}0T$JU&tn zw~Skk5Wr?Ap(S?ysC#AhC}$v8iDTy9u#dF@_qKU%bj;~-NhVoNRU6~N4%eH8CM3eL z4>8s864e4?n^G)8mIt;zL0iGF({ttNi22 zi3ETN&!3Q}?3wNohGm(Ehy1{VzQIqf2a|?#pQ@Jo`jX@})Q5n+pC`}Em*7xu(CEEw-JS^ecX(y3w7i*&JI&)E(>RblGyuP&!oNH_8b%)D`*CaVZsdZ|hb2 zh+?~CkkDAq5K}F&bn9!>9(LmE2BI)CtWfLgTpyr#okSPQDL2nXTPI(ont$OIM3YdN ztK|mCvGcn*hdum3miyWtJz{|{KNjm3`6Wag@_VtCaTby$g<+`nCwbjR2yw-E1nOXy(5LvSsI^CAh`1OvRP5 zm4--Gx8E(?5IPN{V~O(7>XahYQUdc`jeAz8`5J$C83jGwd!`EktYCEfW~Ne9RFj6r zQDPPO`8x$W5DEdrTq`2uGO7d0Lg#yi)A7T7_~l>BGauPSS-e?tlNU#5XxRp}H{fJr zYX&$OFD#cisL#O)O46VR~`;KhkC@&=kz_Q~r3cg-$lKSaUULS>|p9YGOs3 zFJ)QjmdWH!>nT>O(HG?>s;HV<^*hciuvaZA2%zJ3QwdsKZxy1PeWmTM{KC{ZHMS(O zR{6Z%58Q_BFQThj6#~p0{RX1wEUSE=&oui!S*_3cMP(WEkkF8Dn!eZ{8PbJ{FUT=A z*T9>LLqTv-mV4gwL#y4{+tj1$J{QuVc++V@2ddm*uOA_V*A?DOSfIuEhoKl}Gch)k z6&jk0Gyg%w*Tx{s@654JtU^Y^SJ^76I}Y%(&CsEi{*M`HTH1%T;@Mw*WsWmQPBjVd zG!!u(9?FKI7KMYq2ulY(9)PRvJ9`cH-0+2ms$=*Ajaq+^P9iR|{}RYKXZuKd5haDY zvzyJUi`a2p-L?MAiH;(Dmv+=JsP|!q>nZ@wi8z+zAUAx;oi2Oj^rc02J9>So)fOJ~ zt*~@WmDF-h&S+H0A_h*_jQQpsTnrLFdp{OYMBG^-9`VB;@AfLo;skfeN2;OrV=BI1 z{;Ln?^8kHkv4Fg_BajS23oHDVZ6sR3aG?}qIqR&9bKS|N9teb9u)U`}&B$Qs!RRQ4 z7CgAP-h2?Tlx33|R6t}!{L&jYuIM~LYP!ttvts(CB4JocP2T{0LV!GA=9vuG;IgC+ z={&cLj2V<$Q$f zZ4uKG0taR^zxx!s*V(yVgd-l5jPk42EU_Pv&`-(h`Qj1uYRQftL9k_o_2B3nk*2s6 zrQHA=EP};Z#^}h9WJXYsB@^YnIo>D?r~of~0%Q-R7`xLVpOi4N?k{X+=8GNoPRE%o zsp8mDx?NtW$0{86t)Ha*QNKc)>g5OUr`}WriS)=lX2lI^7EMJOoZW)zECXz(rf5Z@ zg0y&sus1RQ!A68ilrm-}uDEDE1nhL6GQA06hRV{tn#t?UN?hqW{@L4`9Y_VF`iDA7WxGW4$Q z^$LuL=hlMALgh`tRdJtGXi1$^)&1-YFQejK>}aW?N|Y$?hc&qb!m^0<`m83ew8;E> z9kRppJenBAPT`Q0_aC9zdfsMe3V0qYEAn8WISD6qkccuVyWcN#aNlM)H3~8X>KbWA zc%ZAk5_aTuTH?^&t(rMh=h%%1JGG@LVWBLKZGji( zZe$QpooXM>n?mZ)?hZ}te&T=VR+8OMoT|Vz7ZIUH`@W