oshdb extraction code

scripts/oshdb/pom.xml

+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.heigit.ohsome.oshdb</groupId>
+  <artifactId>gyrt-ijgi-paper</artifactId>
+  <version>0.0.1-SNAPSHOT</version>
+  
+  	<properties>
+		<java.version>11</java.version>
+		<maven.compiler.source>${java.version}</maven.compiler.source>
+		<maven.compiler.target>${java.version}</maven.compiler.target>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.heigit.bigspatialdata</groupId>
+			<artifactId>oshdb-api</artifactId>
+			<version>0.5.10</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.wololo</groupId>
+			<artifactId>jts2geojson</artifactId>
+			<version>0.14.3</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/com.beust/jcommander -->
+		<dependency>
+			<groupId>com.beust</groupId>
+			<artifactId>jcommander</artifactId>
+			<version>1.78</version>
+		</dependency>
+
+	</dependencies>
+	
+		<build>
+		<plugins>
+			<plugin>
+				<groupId>org.codehaus.mojo</groupId>
+				<artifactId>exec-maven-plugin</artifactId>
+				<version>3.0.0</version>
+				<executions>
+					<execution>
+						<goals>
+							<goal>java</goal>
+						</goals>
+					</execution>
+				</executions>
+				<configuration>
+					<mainClass>org.heigit.ohsome.oshdb.gyrt.Extract</mainClass>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+	
+</project>
\ No newline at end of file

scripts/oshdb/src/main/java/org/heigit/ohsome/oshdb/gyrt/Completeness.java

+package org.heigit.ohsome.oshdb.gyrt;
+
+import static com.google.common.collect.Iterators.peekingIterator;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.math.BigDecimal;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.PriorityQueue;
+import java.util.function.Function;
+import org.heigit.bigspatialdata.oshdb.osm.OSMType;
+import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
+import org.locationtech.jts.algorithm.distance.DiscreteHausdorffDistance;
+import org.locationtech.jts.geom.Geometry;
+import org.wololo.geojson.Feature;
+import org.wololo.geojson.FeatureCollection;
+import org.wololo.geojson.GeoJSONFactory;
+import org.wololo.jts2geojson.GeoJSONReader;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.PeekingIterator;
+
+import static org.heigit.ohsome.oshdb.gyrt.Slum.*;
+
+/*
+  
+ */
+public class Completeness {
+  public static final DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_DATE_TIME;
+
+  public static void main(String[] args) {
+    Path extracts = Paths.get("data");
+
+    List<Slum> slums = ALL;
+    for (Slum slum : slums) {
+
+      Map<String, Function<Geometry, Double>> measures = Map.of( //
+          "building", Geo::areaOf, //
+          "highway", Geo::lengthOf //
+      );
+
+      LocalDateTime startState = LocalDateTime.parse(slum.stage1.beginTimestamp.toString(), timeFormatter);
+      LocalDateTime endState = LocalDateTime.parse(slum.stage3.endTimestamp.toString(), timeFormatter);
+
+      measures.forEach((key, measure) -> {
+        Path extract = extracts.resolve(slum.name + "_" + key + "_2010-01-01T00:00:00Z_2020-01-01T00:00:00Z.geojson");
+        String outputPath = "completness_" + slum.name + "_" + key + "_v2.csv";
+        System.out.println(outputPath);
+        try (PrintStream out = new PrintStream(outputPath)) {
+          Completeness.run(extract, startState, endState, measure, out);
+        } catch (Exception e) {
+          throw new RuntimeException(e);
+        }
+      });
+    }
+  }
+
+  private static void run(Path extract, LocalDateTime startState, LocalDateTime endState,
+      Function<Geometry, Double> measure, PrintStream out) throws IOException {
+    Completeness completeness = new Completeness(startState, endState, measure);
+    completeness.run(extract, out);
+  }
+
+  private final GeoJSONReader reader = new GeoJSONReader();
+
+  private final LocalDateTime startState;
+  private final LocalDateTime endState;
+  private final Function<Feature, BigDecimal> measure;
+
+  private int numBuildingsInFinalStage = 0;
+  private int wrongBuildings = 0;
+  private BigDecimal measureInFinalStage = new BigDecimal(0.0);
+  private BigDecimal distanceToFinalStage = new BigDecimal(0.0);
+  private BigDecimal wrongMeasure = new BigDecimal(0.0);
+
+  private int modCreation = 0;
+  private int modDeletion = 0;
+  private int modTagChange = 0;
+  private int modGeomChange = 0;
+  private int modTagGeomChange = 0;
+
+
+  public Completeness(LocalDateTime startState, LocalDateTime endState, Function<Geometry, Double> measure) {
+    this.startState = startState;
+    this.endState = endState;
+    this.measure = (feature) -> new BigDecimal(measure.apply(reader.read(feature.getGeometry()))
+        .doubleValue());
+  }
+
+  public void run(Path extract, PrintStream out) throws IOException {
+    List<Feature> features = readFeatures(extract);
+
+    System.out.println(Iterables.size(features));
+
+    features.sort((a, b) -> validFrom(a).compareTo(validFrom(b)));
+
+    print(features, out);
+  }
+
+  private void print(List<Feature> events, PrintStream out) {
+    LocalDateTime k = LocalDateTime.parse("2010-01-01T00:00Z", timeFormatter).truncatedTo(ChronoUnit.HOURS);
+    LocalDateTime kPlus1 = k.plusHours(1);
+
+    PeekingIterator<Feature> itr = peekingIterator(events.iterator());
+    PriorityQueue<Feature> active = new PriorityQueue<>((a, b) -> validTo(a).compareTo(validTo(b)));
+
+    printHeader(out);
+    while (itr.hasNext()) {
+      modCreation = modDeletion = modTagChange = modGeomChange = modTagGeomChange = 0;
+
+      while (itr.hasNext() && validTo(itr.peek()).isBefore(k)) {
+        itr.next();
+      }
+
+      removeNonActive(active, kPlus1);
+
+      while (itr.hasNext() && validFrom(itr.peek()).isBefore(kPlus1) && validTo(itr.peek()).isAfter(k)) {
+        Feature feature = itr.next();
+        if (validTo(feature).compareTo(kPlus1) <= 0) {
+          if (feature.getProperties()
+              .containsKey("@creation")) {
+            modCreation++;
+          } else if (feature.getProperties()
+              .containsKey("@tag-change")) {
+            if (feature.getProperties()
+                .containsKey("@geometry-change")) {
+              modTagGeomChange++;
+            } else {
+              modTagChange++;
+            }
+          } else if (feature.getProperties()
+              .containsKey("@geometry-change")) {
+            modGeomChange++;
+          }
+          continue;
+        }
+        add(active, feature);
+      }
+      if (k.compareTo(startState) >= 0) {
+
+        print(k, kPlus1, out);
+      }
+      k = kPlus1;
+      kPlus1 = k.plusHours(1);
+    }
+
+    while (!active.isEmpty()) {
+      modCreation = modDeletion = modTagChange = modGeomChange = modTagGeomChange = 0;
+      removeNonActive(active, kPlus1);
+      print(k, kPlus1, out);
+
+      k = kPlus1;
+      kPlus1 = k.plusHours(1);
+
+      if (k.compareTo(endState) >= 0)
+        break;
+    }
+  }
+
+  private List<Feature> readFeatures(Path path) throws IOException {
+    String json = new String(Files.readAllBytes(path));
+    FeatureCollection featureCollection = (FeatureCollection) GeoJSONFactory.create(json);
+    return getEvents(Arrays.asList(featureCollection.getFeatures()));
+  }
+
+  private List<Feature> getEvents(List<Feature> features) {
+    List<Feature> events = new ArrayList<>();
+    final PeekingIterator<Feature> upstream = peekingIterator(features.iterator());
+
+    while (upstream.hasNext()) {
+      Feature feature = upstream.peek();
+      OSMId osmId = osmId(feature);
+      while (upstream.hasNext() && osmId(upstream.peek()).equals(osmId)) {
+        feature = upstream.next();
+        if (validFrom(feature).isBefore(endState)) {
+          events.add(feature);
+        }
+      }
+    }
+
+    return events;
+  }
+
+
+
+  private void removeNonActive(PriorityQueue<Feature> active, LocalDateTime k) {
+    while (!active.isEmpty() && validTo(active.peek()).compareTo(k) < 0) {
+      Feature feature = active.poll();
+      OSMId osmId = osmId(feature);
+
+      if (feature.getProperties()
+          .containsKey("@deletion")) {
+        modDeletion++;
+      }
+
+      BigDecimal value = measure.apply(feature);
+      boolean inFinalStage = 
+          feature.getProperties().containsKey("@stage-3") &&
+          feature.getProperties().containsKey("@stage-after");
+
+      if (inFinalStage) {
+        numBuildingsInFinalStage -= 1;
+        measureInFinalStage = measureInFinalStage.subtract(value);
+
+//        double distance = DiscreteHausdorffDistance.distance(geom(feature), geom(endStage));
+//        distanceToFinalStage = distanceToFinalStage.subtract(new BigDecimal(distance));
+
+      } else {
+        wrongBuildings -= 1;
+        wrongMeasure = wrongMeasure.subtract(value);
+      }
+    }
+  }
+
+  private void add(PriorityQueue<Feature> active, Feature feature) {
+    active.add(feature);
+
+    if (feature.getProperties()
+        .containsKey("@creation")) {
+      modCreation++;
+    } else if (feature.getProperties()
+        .containsKey("@tag-change")) {
+      if (feature.getProperties()
+          .containsKey("@geometry-change")) {
+        modTagGeomChange++;
+      } else {
+        modTagChange++;
+      }
+    } else if (feature.getProperties()
+        .containsKey("@geometry-change")) {
+      modGeomChange++;
+    }
+
+    BigDecimal value = measure.apply(feature);
+    //Feature endStage = endStageFeatures.get(osmId(feature));
+    boolean inFinalStage = 
+        feature.getProperties().containsKey("@stage-3") &&
+        feature.getProperties().containsKey("@stage-after");
+
+    if (inFinalStage) {
+      OSMId osmId = osmId(feature);
+
+      numBuildingsInFinalStage += 1;
+      measureInFinalStage = measureInFinalStage.add(value);
+
+      //double distance = DiscreteHausdorffDistance.distance(geom(feature), geom(endStage));
+      //distanceToFinalStage = distanceToFinalStage.add(new BigDecimal(distance));
+
+    } else {
+      wrongBuildings += 1;
+      wrongMeasure = wrongMeasure.add(value);
+    }
+  }
+
+  private Geometry geom(Feature feature) {
+    return reader.read(feature.getGeometry());
+  }
+
+  private void printHeader(PrintStream out) {
+    out.println("k, k_plus1" + ", final_stage" //
+        + ", wrong" //
+        + ", final_stage_measure" //
+        + ", wrong_measure" //
+        + ", distance" //
+        + ", total_num_objects" //
+        + ", modCreation" //
+        + ", modDeletion" //
+        + ", modTagChange" //
+        + ", modGeomChange" //
+        + ", modTagGeomChange");
+  }
+
+  private void print(LocalDateTime k, LocalDateTime kPlus1, PrintStream out) {
+    out.println(k + "," + kPlus1 // 
+        + "," + numBuildingsInFinalStage // 
+        + "," + wrongBuildings //
+        + "," + measureInFinalStage.doubleValue() // 
+        + "," + wrongMeasure.doubleValue() //
+        + "," + distanceToFinalStage //
+        + "," + (numBuildingsInFinalStage + wrongBuildings) + "," + modCreation //
+        + "," + modDeletion //
+        + "," + modTagChange //
+        + "," + modGeomChange //
+        + "," + modTagGeomChange);
+  }
+
+
+  private static class OSMId {
+    public final OSMType type;
+    public final long id;
+
+    public OSMId(OSMType type, long id) {
+      this.type = type;
+      this.id = id;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(id, type);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj)
+        return true;
+      if (!(obj instanceof OSMId))
+        return false;
+      OSMId other = (OSMId) obj;
+      return id == other.id && type == other.type;
+    }
+  }
+
+  private static OSMId osmId(Feature f) {
+    OSMType type = OSMType.valueOf((String) f.getProperties()
+        .get("@osmType"));
+    Object obj = f.getProperties()
+        .get("@osmId");
+    long id;
+    if (obj instanceof Integer) {
+      id = ((Integer) obj).intValue();
+    } else if (obj instanceof Long) {
+      id = ((Long) obj).longValue();
+    } else {
+      throw new RuntimeException("could not convert osmId! (" + obj + ")");
+    }
+    return new OSMId(type, id);
+  }
+
+  private static LocalDateTime validFrom(Feature f) {
+    return LocalDateTime.parse((String) f.getProperties()
+        .get("@validFrom"), timeFormatter);
+  }
+
+  private static LocalDateTime validTo(Feature f) {
+    return LocalDateTime.parse((String) f.getProperties()
+        .get("@validTo"), timeFormatter);
+  }
+
+}

scripts/oshdb/src/main/java/org/heigit/ohsome/oshdb/gyrt/Config.java

+package org.heigit.ohsome.oshdb.gyrt;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import org.heigit.bigspatialdata.oshdb.util.time.OSHDBTimestamps;
+
+public class Config {
+  
+  public static final List<Slum> SLUMS = Slum.ALL;
+  public static final List<String> KEYS = List.of("building", "highway"); 
+  
+  public static final String isoDateStart = "2010-01-01T00:00:00Z";
+  public static final String isoDateEnd =   "2020-01-01T00:00:00Z";
+  
+  public static final OSHDBTimestamps fullHistory= new OSHDBTimestamps(isoDateStart, isoDateEnd);
+  
+  public static final Path geojsons = Paths.get("sites");
+  
+  public static final Path igniteCfg = Paths.get("PATH_TO_IGNITE_CONFIGURATION");
+  public static final String prefix = "PREFIX";
+  public static final String keytableUrl = "URL_TO_KEYTABLE_DATABASE";
+  
+  
+  
+}

scripts/oshdb/src/main/java/org/heigit/ohsome/oshdb/gyrt/Extract.java

+package org.heigit.ohsome.oshdb.gyrt;
+
+import static org.heigit.ohsome.oshdb.gyrt.util.OSHDBUtil.connectToIgnite;
+import static org.heigit.ohsome.oshdb.gyrt.util.OSHDBUtil.loadGeoJSON;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableFunction;
+import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
+import org.heigit.bigspatialdata.oshdb.osm.OSMEntity;
+import org.heigit.bigspatialdata.oshdb.osm.OSMType;
+import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
+import org.heigit.bigspatialdata.oshdb.util.OSHDBTagKey;
+import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
+import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
+import org.heigit.bigspatialdata.oshdb.util.taginterpreter.DefaultTagInterpreter;
+import org.heigit.bigspatialdata.oshdb.util.taginterpreter.TagInterpreter;
+import org.heigit.bigspatialdata.oshdb.util.tagtranslator.OSMTag;
+import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
+import org.heigit.ohsome.oshdb.gyrt.util.ThrowingConsumer;
+import org.heigit.ohsome.oshdb.gyrt.util.OSHDBUtil.OSHDBWrapper;
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.Polygonal;
+import org.locationtech.jts.geom.PrecisionModel;
+import org.locationtech.jts.precision.GeometryPrecisionReducer;
+import org.wololo.geojson.Feature;
+import org.wololo.jts2geojson.GeoJSONWriter;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import static org.heigit.ohsome.oshdb.gyrt.Slum.*;
+
+public class Extract {
+
+  public static void main(String[] args) throws Exception {
+    Path data = Paths.get("/home/rtroilo/godwin-gyrt/paper/data");
+    extract(data);
+  }
+  
+  public static void extract(Path outputPath) throws Exception {
+    Files.createDirectories(outputPath);
+    System.out.println("extracting >>>");
+    Stopwatch stopwatch = Stopwatch.createStarted();
+    connectToIgnite(Config.igniteCfg, Config.prefix, Config.keytableUrl, run(outputPath));
+    System.out.println("<<< " + stopwatch);
+  }
+
+  public static ThrowingConsumer<OSHDBWrapper, Exception> run(Path outputDir) {
+    return (oshdb) -> {
+      TagTranslator tagTranslator = oshdb.getDefaultTagTranslator();
+      TagInterpreter tagInterpreter = new DefaultTagInterpreter(tagTranslator);
+
+      Map<String, OSHDBTagKey> map = new HashMap<>();
+
+      for(String key : Config.KEYS) {
+        map.put(key, tagTranslator.getOSHDBTagKeyOf(key));
+      }
+      
+      List<Slum> slums = Config.SLUMS;
+      for (Slum slum : slums) {
+        Geometry site = loadGeoJSON(Config.geojsons.resolve(slum.name + ".geojson"))[0];
+        for (Map.Entry<String, OSHDBTagKey> key : map.entrySet()) {
+          Path outputFile = outputDir.resolve(slum.name +"_"+key.getKey().toString()+"_"+Config.isoDateStart+"_"+Config.isoDateEnd+".geojson");
+          try (PrintWriter output = new PrintWriter(Files.newBufferedWriter(outputFile))) {
+             System.out.println(outputFile);
+             List<Feature> features = oshdb.getContributionView() //
+                .areaOfInterest((Geometry & Polygonal) site) //
+                .timestamps(Config.fullHistory) //
+                .osmTag(key.getKey()) //
+                .groupByEntity() //
+                .map(Extractor.map(slum, key.getValue(), tagInterpreter)) //
+                .flatMap(f -> f)
+                .collect();
+             
+             features.forEach(feature -> {
+               Map<String, Object> props = feature.getProperties();
+               OSHDBTag[] tags = (OSHDBTag[]) props.remove("@tags");
+               for (OSHDBTag tag : tags) {
+                 OSMTag osmTag = tagTranslator.getOSMTagOf(tag);
+                 props.put(osmTag.getKey(), osmTag.getValue());
+               }
+             });
+             
+
+            output.print("{");
+            output.print(
+                "\"attribution\" : { \"url\" : \"https://ohsome.org/copyrights\",\"text\" : \"© OpenStreetMap contributors\"},");
+            output.print("\"apiVersion\" : \"1.0\",");
+            output.print("\"type\" : \"FeatureCollection\",");
+            output.println("\"features\" : [");
+            
+            Iterator<Feature> itr = features.iterator();
+          
+            while (itr.hasNext()) {
+              output.print(itr.next());
+              if (itr.hasNext()) {
+                output.println(",");
+              }
+            }
+            
+            output.println();
+            output.print("]");
+            output.println("}");
+          }
+        }
+      }
+    };
+  }
+
+
+  public static class Extractor implements Serializable {
+
+    private static final long serialVersionUID = 2L;
+    private final Slum slum;
+    private final OSHDBTagKey key;
+    private final GeoJSONWriter writer = new GeoJSONWriter();
+    private final PrecisionModel pm = new PrecisionModel(10000000.0);
+    private final GeometryPrecisionReducer precisionReducer = new GeometryPrecisionReducer(pm);
+    
+
+    private Extractor(Slum slum, OSHDBTagKey key, TagInterpreter tagInterpreter) {
+      this.slum = slum;
+      this.key = key;
+    }
+
+    public static SerializableFunction<List<OSMContribution>, List<Feature>> map(Slum slum, OSHDBTagKey key,
+        TagInterpreter tagInterpreter) {
+      Extractor extractor = new Extractor(slum, key, tagInterpreter);
+      return extractor::map;
+    }
+
+    public List<Feature> map(List<OSMContribution> contribs) {
+      List<Feature> features = new ArrayList<>(contribs.size());