diff --git a/bikenetlib/_overpass.py b/bikenetlib/_overpass.py new file mode 100644 index 0000000..7aec98f --- /dev/null +++ b/bikenetlib/_overpass.py @@ -0,0 +1,10 @@ +"""Overpass settings for `bikenetlib`. Do not change.""" + +_op_ref = "BikeNetLib Python package (https://github.com/BikeNetKit/BikeNetLib)" +_op_url = 'https://overpass.bikenetkit.org/api' + +import osmnx.settings as oxs +oxs.http_user_agent = _op_ref +oxs.http_op_referer = _op_ref +oxs.overpop_ass_url = _op_url +oxs.overpass_rate_limit = True \ No newline at end of file diff --git a/bikenetlib/io.py b/bikenetlib/io.py index 56d0d90..c9d2747 100644 --- a/bikenetlib/io.py +++ b/bikenetlib/io.py @@ -7,9 +7,99 @@ silent = False +import os import re from tqdm import tqdm -from . import constants, osm +from . import constants, osm, _overpass + + +def export_gdfs( + gdfs, + filename='exported_data', + path="./", + format='gpkg', + overwrite=True, + crs='4326', + bar=False + ): + """Export GeoDataFrames as geojson, gpkg, or shp file(s). + + If crs=='4326' and format=='geojson', export with RFC7946. + + Parameters + ---------- + gdfs : dict + Dictionary with the layer names as keys with dict values that contain + gdf and single filename. Structure: + { + 'layername1': {'gdf': gdf1, 'filename_single':str}, + 'layername2': {'gdf': gdf2, 'filename_single':str}, + ... + } + `filename_single` is relevant for geojson and shp files which ignore + `filename` and export each gdf as individual files. `layername` is + relevant for gpkg which exports all gdfs merged into one file + `filename` as their own layers. + filename : str, default 'exported_data' + Name of exported file, if exported as gpkg. + path : str, default './' + Export path. + format : {'gpkg', 'geojson', 'shp'}, default 'gpkg' + Export file format. + overwrite : bool, default True + Decision whether to overwrite existing files. + crs : str, default '4326' + Coordinate Reference System of exported data. + bar : bool, default False + Decision whether to show a progress bar. + + Returns + ------- + bool + Returns True if all files have been written. If overwrite is set to + False and at least one file already existed, returns False. + """ + os.makedirs(path, exist_ok=True) + + format_driver = {'gpkg': "GPKG", 'geojson': "GeoJSON", 'shp': "ESRI Shapefile"} + d = format_driver[format] + + if bar: progress_bar = initialize_progress_bar("Exporting data", len(gdfs), "file") + + if format == 'gpkg': # Export as one file with layers + f = path+filename+'.'+format + # Check if file exists, and remove if overwrite + if os.path.exists(f): + if not overwrite: + return False + else: + os.remove(f) + # Export one gpkg with layers + for layername, gdfinfo in gdfs.items(): + gdfinfo['gdf'].to_file(f, driver=d, layer=layername) + if bar: progress_bar.update(1) + + elif format == 'geojson' or format == 'shp': # Export as individual files + # Check if files exists, and remove if overwrite + for gdfinfo in gdfs.values(): + f = path+gdfinfo['filename_single']+'.'+format + if os.path.exists(f): + if not overwrite: + return False + else: + os.remove(f) + # Export individual files + for layername, gdfinfo in gdfs.items(): + f = path+gdfinfo['filename_single']+'.'+format + if format == 'geojson' and crs == '4326': + gdfinfo['gdf'].to_file(f, driver=d, RFC7946="YES") + else: + gdfinfo['gdf'].to_file(f, driver=d) + if bar: progress_bar.update(1) + + if bar: progress_bar.close() + return True + def slugify(s): """Slugify a string. diff --git a/bikenetlib/utils.py b/bikenetlib/utils.py index d0dc6d1..8984f41 100644 --- a/bikenetlib/utils.py +++ b/bikenetlib/utils.py @@ -5,7 +5,7 @@ import geopandas as gpd import networkx as nx import osmnx as ox -from . import constants, osm +from . import constants, osm, _overpass def assign_pbi_to_edges(g): diff --git a/docs/consolidate.md b/docs/consolidate.md index c808e54..7c61bc3 100644 --- a/docs/consolidate.md +++ b/docs/consolidate.md @@ -16,6 +16,7 @@ Input/Output like downloads, imports, exports, but also format conversions and u ## Multi-use - X `slugify()` - X `initialize_progress_bar()`: require tqdm +- X `export_gdfs()` - `import_network()`, `prepare_nodes_edges()`: align gbn-fbn - `download_network()`: align gbn-fbn - `_resolve_crs_calculations()` diff --git a/tests/test_data/oelde_boundary.geojson b/tests/test_data/oelde_boundary.geojson new file mode 100644 index 0000000..2bdda70 --- /dev/null +++ b/tests/test_data/oelde_boundary.geojson @@ -0,0 +1,7 @@ +{ +"type": "FeatureCollection", +"name": "oelde-growbikenet-city_boundary", +"features": [ +{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[8.1751926,51.760428],[8.1933235,51.7618463],[8.1946563,51.7621123],[8.2521232,51.7782795],[8.2595564,51.7913977],[8.2597579,51.7935527],[8.2594274,51.7956448],[8.2586074,51.7967077],[8.2151636,51.8473703],[8.1985841,51.8625767],[8.1567163,51.8970066],[8.1490641,51.8960361],[8.1441867,51.8948118],[8.128341,51.8897426],[8.0847223,51.8360191],[8.0850733,51.8209566],[8.0857182,51.8197211],[8.1165409,51.7742195],[8.120826,51.7704191],[8.1392967,51.7650706],[8.1570866,51.762327],[8.174567,51.7604912],[8.1751926,51.760428]]]}} +] +} diff --git a/tests/test_data/oelde_export.gpkg b/tests/test_data/oelde_export.gpkg new file mode 100644 index 0000000..22343ac Binary files /dev/null and b/tests/test_data/oelde_export.gpkg differ diff --git a/tests/test_data/oelde_seed_points.geojson b/tests/test_data/oelde_seed_points.geojson new file mode 100644 index 0000000..8c289c3 --- /dev/null +++ b/tests/test_data/oelde_seed_points.geojson @@ -0,0 +1,12 @@ +{ +"type": "FeatureCollection", +"name": "oelde_seed_points", +"features": [ +{ "type": "Feature", "properties": { "fid": 1 }, "geometry": { "type": "Point", "coordinates": [ 8.1529, 51.84924 ] } }, +{ "type": "Feature", "properties": { "fid": 2 }, "geometry": { "type": "Point", "coordinates": [ 8.1429, 51.82924 ] } }, +{ "type": "Feature", "properties": { "fid": 3 }, "geometry": { "type": "Point", "coordinates": [ 8.1329, 51.81924 ] } }, +{ "type": "Feature", "properties": { "fid": 4 }, "geometry": { "type": "Point", "coordinates": [ 8.1529, 51.81924 ] } }, +{ "type": "Feature", "properties": { "fid": 5 }, "geometry": { "type": "Point", "coordinates": [ 8.1429, 51.81924 ] } }, +{ "type": "Feature", "properties": { "fid": 6 }, "geometry": { "type": "Point", "coordinates": [ 8.1329, 51.82924 ] } } +] +} diff --git a/tests/test_io.py b/tests/test_io.py index cfd0208..ef32b5b 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,12 +1,128 @@ import pytest from tqdm import tqdm from bikenetlib import constants, io +import geopandas as gpd +import os +import shutil +from pandas.testing import assert_frame_equal from bikenetlib.io import ( + export_gdfs, slugify, initialize_progress_bar, ) +path_test_data = './tests/test_data/' + +# export_gdfs +@pytest.fixture +def oelde_gdfs(): + gdfs = {} + gdfs["Seed points"] = { + 'gdf': gpd.read_file(path_test_data+"oelde_seed_points.geojson"), + 'filename_single': 'oelde_seed_points', + } + gdfs["City boundary"] = { + 'gdf': gpd.read_file(path_test_data+"oelde_boundary.geojson"), + 'filename_single': 'oelde_boundary', + } + return gdfs + +@pytest.fixture +def validation_oelde_gpkg_layer(): + gdf = gpd.read_file(path_test_data+"oelde_export.gpkg", layer="Seed points") + return gdf + +def test_export_gdfs(oelde_gdfs, validation_oelde_gpkg_layer): + path_test_outputs = './tests/test_outputs/' + filename = 'oelde_export' + + format = 'gpkg' + if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs): + shutil.rmtree(path_test_outputs) + export_gdfs( + oelde_gdfs, + filename=filename, + path=path_test_outputs, + format=format, + bar=False, + ) + assert_frame_equal( + validation_oelde_gpkg_layer, + gpd.read_file(path_test_outputs+"oelde_export.gpkg", layer="Seed points"), + check_dtype=False, + ) + # Test overwrite + o = export_gdfs( + oelde_gdfs, + filename=filename, + path=path_test_data, # ! + format=format, + bar=False, + overwrite=False, # ! + ) + assert o == False + + format = 'geojson' + if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs): + shutil.rmtree(path_test_outputs) + export_gdfs( + oelde_gdfs, + filename=filename, + path=path_test_outputs, + format=format, + bar=False, + ) + assert_frame_equal( + gpd.read_file(path_test_data+"oelde_seed_points.geojson"), + gpd.read_file(path_test_outputs+"oelde_seed_points.geojson"), + check_dtype=False, + ) + assert_frame_equal( + gpd.read_file(path_test_data+"oelde_boundary.geojson"), + gpd.read_file(path_test_outputs+"oelde_boundary.geojson"), + check_dtype=False, + ) + # Test overwrite + o = export_gdfs( + oelde_gdfs, + filename=filename, + path=path_test_data, # ! + format=format, + bar=False, + overwrite=False, # ! + ) + assert o == False + + format = 'shp' + if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs): + shutil.rmtree(path_test_outputs) + export_gdfs( + oelde_gdfs, + filename=filename, + path=path_test_outputs, + format=format, + bar=False, + ) + assert_frame_equal( + gpd.read_file(path_test_data+"oelde_seed_points.geojson"), + gpd.read_file(path_test_outputs+"oelde_seed_points.shp"), + check_dtype=False, + ) + # To do: Some weird FID issue when loading a single-row shape file. + # shp = gpd.read_file(path_test_outputs+"oelde_boundary.shp") + # del shp['FID'] + # assert_frame_equal( + # gpd.read_file(path_test_data+"oelde_boundary.geojson"), + # shp, + # check_dtype=False, + # ) + + # Clean up + if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs): + shutil.rmtree(path_test_outputs) + + # slugify @pytest.fixture def city_names(): @@ -23,6 +139,7 @@ def test_slugify(city_names, city_names_slugified): assert slugify(n) == ns +# initialize_progress_bar @pytest.fixture def progress_bar_string(): return "Test string"