Feature/add overpass and export gdfs

bikenetlib/_overpass.py

@@ -0,0 +1,10 @@
+"""Overpass settings for `bikenetlib`. Do not change."""
+
+_op_ref = "BikeNetLib Python package (https://github.com/BikeNetKit/BikeNetLib)"
+_op_url = 'https://overpass.bikenetkit.org/api'
+
+import osmnx.settings as oxs
+oxs.http_user_agent = _op_ref
+oxs.http_op_referer = _op_ref
+oxs.overpop_ass_url = _op_url
+oxs.overpass_rate_limit = True

bikenetlib/io.py

@@ -7,9 +7,99 @@
 
 silent = False
 
+import os
 import re
 from tqdm import tqdm
-from . import constants, osm
+from . import constants, osm, _overpass
+
+
+def export_gdfs(
+        gdfs,
+        filename='exported_data',
+        path="./",
+        format='gpkg',
+        overwrite=True,
+        crs='4326',
+        bar=False
+    ):
+    """Export GeoDataFrames as geojson, gpkg, or shp file(s).
+
+    If crs=='4326' and format=='geojson', export with RFC7946.
+
+    Parameters
+    ----------
+    gdfs : dict
+        Dictionary with the layer names as keys with dict values that contain 
+        gdf and single filename. Structure:
+        {
+            'layername1': {'gdf': gdf1, 'filename_single':str},
+            'layername2': {'gdf': gdf2, 'filename_single':str},
+            ...
+        }
+        `filename_single` is relevant for geojson and shp files which ignore
+        `filename` and export each gdf as individual files. `layername` is 
+        relevant for gpkg which exports all gdfs merged into one file 
+        `filename` as their own layers.
+    filename : str, default 'exported_data'
+        Name of exported file, if exported as gpkg.
+    path : str, default './'
+        Export path.
+    format : {'gpkg', 'geojson', 'shp'}, default 'gpkg'
+        Export file format.
+    overwrite : bool, default True
+        Decision whether to overwrite existing files.
+    crs : str, default '4326'
+        Coordinate Reference System of exported data.
+    bar : bool, default False
+        Decision whether to show a progress bar.
+
+    Returns
+    -------
+    bool
+        Returns True if all files have been written. If overwrite is set to 
+        False and at least one file already existed, returns False. 
+    """
+    os.makedirs(path, exist_ok=True)
+
+    format_driver = {'gpkg': "GPKG", 'geojson': "GeoJSON", 'shp': "ESRI Shapefile"}
+    d = format_driver[format]
+
+    if bar: progress_bar = initialize_progress_bar("Exporting data", len(gdfs), "file")
+
+    if format == 'gpkg': # Export as one file with layers
+        f = path+filename+'.'+format
+        # Check if file exists, and remove if overwrite
+        if os.path.exists(f):
+            if not overwrite:
+                return False
+            else:
+                os.remove(f)
+        # Export one gpkg with layers
+        for layername, gdfinfo in gdfs.items():
+            gdfinfo['gdf'].to_file(f, driver=d, layer=layername)
+            if bar: progress_bar.update(1)
+
+    elif format == 'geojson' or format == 'shp': # Export as individual files
+        # Check if files exists, and remove if overwrite
+        for gdfinfo in gdfs.values():
+            f = path+gdfinfo['filename_single']+'.'+format
+            if os.path.exists(f):
+                if not overwrite:
+                    return False
+                else:
+                    os.remove(f)
+        # Export individual files
+        for layername, gdfinfo in gdfs.items():
+            f = path+gdfinfo['filename_single']+'.'+format
+            if format == 'geojson' and crs == '4326':
+                gdfinfo['gdf'].to_file(f, driver=d, RFC7946="YES")
+            else:
+                gdfinfo['gdf'].to_file(f, driver=d)
+            if bar: progress_bar.update(1)
+
+    if bar: progress_bar.close()
+    return True
+
 
 def slugify(s): 
     """Slugify a string.

bikenetlib/utils.py

@@ -5,7 +5,7 @@
 import geopandas as gpd
 import networkx as nx
 import osmnx as ox
-from . import constants, osm
+from . import constants, osm, _overpass
 
 
 def assign_pbi_to_edges(g):

docs/consolidate.md

@@ -16,6 +16,7 @@ Input/Output like downloads, imports, exports, but also format conversions and u
 ## Multi-use
 - X `slugify()`
 - X `initialize_progress_bar()`: require tqdm
+- X `export_gdfs()`
 - `import_network()`, `prepare_nodes_edges()`: align gbn-fbn
 - `download_network()`: align gbn-fbn
 - `_resolve_crs_calculations()`

tests/test_data/oelde_boundary.geojson

@@ -0,0 +1,7 @@
+{
+"type": "FeatureCollection",
+"name": "oelde-growbikenet-city_boundary",
+"features": [
+{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[8.1751926,51.760428],[8.1933235,51.7618463],[8.1946563,51.7621123],[8.2521232,51.7782795],[8.2595564,51.7913977],[8.2597579,51.7935527],[8.2594274,51.7956448],[8.2586074,51.7967077],[8.2151636,51.8473703],[8.1985841,51.8625767],[8.1567163,51.8970066],[8.1490641,51.8960361],[8.1441867,51.8948118],[8.128341,51.8897426],[8.0847223,51.8360191],[8.0850733,51.8209566],[8.0857182,51.8197211],[8.1165409,51.7742195],[8.120826,51.7704191],[8.1392967,51.7650706],[8.1570866,51.762327],[8.174567,51.7604912],[8.1751926,51.760428]]]}}
+]
+}

tests/test_data/oelde_seed_points.geojson

@@ -0,0 +1,12 @@
+{
+"type": "FeatureCollection",
+"name": "oelde_seed_points",
+"features": [
+{ "type": "Feature", "properties": { "fid": 1 }, "geometry": { "type": "Point", "coordinates": [ 8.1529, 51.84924 ] } },
+{ "type": "Feature", "properties": { "fid": 2 }, "geometry": { "type": "Point", "coordinates": [ 8.1429, 51.82924 ] } },
+{ "type": "Feature", "properties": { "fid": 3 }, "geometry": { "type": "Point", "coordinates": [ 8.1329, 51.81924 ] } },
+{ "type": "Feature", "properties": { "fid": 4 }, "geometry": { "type": "Point", "coordinates": [ 8.1529, 51.81924 ] } },
+{ "type": "Feature", "properties": { "fid": 5 }, "geometry": { "type": "Point", "coordinates": [ 8.1429, 51.81924 ] } },
+{ "type": "Feature", "properties": { "fid": 6 }, "geometry": { "type": "Point", "coordinates": [ 8.1329, 51.82924 ] } }
+]
+}

tests/test_io.py

@@ -1,12 +1,128 @@
 import pytest
 from tqdm import tqdm
 from bikenetlib import constants, io
+import geopandas as gpd
+import os
+import shutil
+from pandas.testing import assert_frame_equal
 
 from bikenetlib.io import (
+    export_gdfs,
     slugify,
     initialize_progress_bar,
 )
 
+path_test_data = './tests/test_data/'
+
+# export_gdfs
+@pytest.fixture
+def oelde_gdfs():
+    gdfs = {}
+    gdfs["Seed points"] = {
+        'gdf': gpd.read_file(path_test_data+"oelde_seed_points.geojson"),
+        'filename_single': 'oelde_seed_points',
+    }
+    gdfs["City boundary"] = {
+        'gdf': gpd.read_file(path_test_data+"oelde_boundary.geojson"),
+        'filename_single': 'oelde_boundary',
+    }
+    return gdfs
+
+@pytest.fixture
+def validation_oelde_gpkg_layer():
+    gdf = gpd.read_file(path_test_data+"oelde_export.gpkg", layer="Seed points")
+    return gdf
+
+def test_export_gdfs(oelde_gdfs, validation_oelde_gpkg_layer):
+    path_test_outputs = './tests/test_outputs/'
+    filename = 'oelde_export'
+
+    format = 'gpkg'
+    if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs):
+        shutil.rmtree(path_test_outputs)
+    export_gdfs(
+        oelde_gdfs, 
+        filename=filename, 
+        path=path_test_outputs, 
+        format=format,
+        bar=False,
+    )
+    assert_frame_equal(
+        validation_oelde_gpkg_layer,
+        gpd.read_file(path_test_outputs+"oelde_export.gpkg", layer="Seed points"),
+        check_dtype=False,
+    )
+    # Test overwrite
+    o = export_gdfs(
+        oelde_gdfs, 
+        filename=filename, 
+        path=path_test_data, # !
+        format=format,
+        bar=False,
+        overwrite=False, # !
+    )
+    assert o == False
+
+    format = 'geojson'
+    if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs):
+        shutil.rmtree(path_test_outputs)
+    export_gdfs(
+        oelde_gdfs, 
+        filename=filename, 
+        path=path_test_outputs, 
+        format=format,
+        bar=False,
+    )
+    assert_frame_equal(
+        gpd.read_file(path_test_data+"oelde_seed_points.geojson"),
+        gpd.read_file(path_test_outputs+"oelde_seed_points.geojson"),
+        check_dtype=False,
+    )
+    assert_frame_equal(
+        gpd.read_file(path_test_data+"oelde_boundary.geojson"),
+        gpd.read_file(path_test_outputs+"oelde_boundary.geojson"),
+        check_dtype=False,
+    )
+    # Test overwrite
+    o = export_gdfs(
+        oelde_gdfs, 
+        filename=filename, 
+        path=path_test_data, # !
+        format=format,
+        bar=False,
+        overwrite=False, # !
+    )
+    assert o == False
+
+    format = 'shp'
+    if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs):
+        shutil.rmtree(path_test_outputs)
+    export_gdfs(
+        oelde_gdfs, 
+        filename=filename, 
+        path=path_test_outputs, 
+        format=format,
+        bar=False,
+    )
+    assert_frame_equal(
+        gpd.read_file(path_test_data+"oelde_seed_points.geojson"),
+        gpd.read_file(path_test_outputs+"oelde_seed_points.shp"),
+        check_dtype=False,
+    )
+    # To do: Some weird FID issue when loading a single-row shape file.
+    # shp = gpd.read_file(path_test_outputs+"oelde_boundary.shp")
+    # del shp['FID']
+    # assert_frame_equal(
+    #     gpd.read_file(path_test_data+"oelde_boundary.geojson"),
+    #     shp,
+    #     check_dtype=False,
+    # )
+
+    # Clean up
+    if os.path.exists(path_test_outputs) and os.path.isdir(path_test_outputs):
+        shutil.rmtree(path_test_outputs)
+
+
 # slugify
 @pytest.fixture
 def city_names():
@@ -23,6 +139,7 @@ def test_slugify(city_names, city_names_slugified):
         assert slugify(n) == ns
 
 
+# initialize_progress_bar
 @pytest.fixture
 def progress_bar_string():
     return "Test string"