Verified Commit 1120a4f5 authored by Rafael Troilo's avatar Rafael Troilo
Browse files

oshdb extraction code

parent 2fafdb5e
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.heigit.ohsome.oshdb</groupId>
<artifactId>gyrt-ijgi-paper</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<java.version>11</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.heigit.bigspatialdata</groupId>
<artifactId>oshdb-api</artifactId>
<version>0.5.10</version>
</dependency>
<dependency>
<groupId>org.wololo</groupId>
<artifactId>jts2geojson</artifactId>
<version>0.14.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.beust/jcommander -->
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>1.78</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<goals>
<goal>java</goal>
</goals>
</execution>
</executions>
<configuration>
<mainClass>org.heigit.ohsome.oshdb.gyrt.Extract</mainClass>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
package org.heigit.ohsome.oshdb.gyrt;
import static com.google.common.collect.Iterators.peekingIterator;
import java.io.IOException;
import java.io.PrintStream;
import java.math.BigDecimal;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.PriorityQueue;
import java.util.function.Function;
import org.heigit.bigspatialdata.oshdb.osm.OSMType;
import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
import org.locationtech.jts.algorithm.distance.DiscreteHausdorffDistance;
import org.locationtech.jts.geom.Geometry;
import org.wololo.geojson.Feature;
import org.wololo.geojson.FeatureCollection;
import org.wololo.geojson.GeoJSONFactory;
import org.wololo.jts2geojson.GeoJSONReader;
import com.google.common.collect.Iterables;
import com.google.common.collect.PeekingIterator;
import static org.heigit.ohsome.oshdb.gyrt.Slum.*;
/*
*/
public class Completeness {
public static final DateTimeFormatter timeFormatter = DateTimeFormatter.ISO_DATE_TIME;
public static void main(String[] args) {
Path extracts = Paths.get("data");
List<Slum> slums = ALL;
for (Slum slum : slums) {
Map<String, Function<Geometry, Double>> measures = Map.of( //
"building", Geo::areaOf, //
"highway", Geo::lengthOf //
);
LocalDateTime startState = LocalDateTime.parse(slum.stage1.beginTimestamp.toString(), timeFormatter);
LocalDateTime endState = LocalDateTime.parse(slum.stage3.endTimestamp.toString(), timeFormatter);
measures.forEach((key, measure) -> {
Path extract = extracts.resolve(slum.name + "_" + key + "_2010-01-01T00:00:00Z_2020-01-01T00:00:00Z.geojson");
String outputPath = "completness_" + slum.name + "_" + key + "_v2.csv";
System.out.println(outputPath);
try (PrintStream out = new PrintStream(outputPath)) {
Completeness.run(extract, startState, endState, measure, out);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
}
private static void run(Path extract, LocalDateTime startState, LocalDateTime endState,
Function<Geometry, Double> measure, PrintStream out) throws IOException {
Completeness completeness = new Completeness(startState, endState, measure);
completeness.run(extract, out);
}
private final GeoJSONReader reader = new GeoJSONReader();
private final LocalDateTime startState;
private final LocalDateTime endState;
private final Function<Feature, BigDecimal> measure;
private int numBuildingsInFinalStage = 0;
private int wrongBuildings = 0;
private BigDecimal measureInFinalStage = new BigDecimal(0.0);
private BigDecimal distanceToFinalStage = new BigDecimal(0.0);
private BigDecimal wrongMeasure = new BigDecimal(0.0);
private int modCreation = 0;
private int modDeletion = 0;
private int modTagChange = 0;
private int modGeomChange = 0;
private int modTagGeomChange = 0;
public Completeness(LocalDateTime startState, LocalDateTime endState, Function<Geometry, Double> measure) {
this.startState = startState;
this.endState = endState;
this.measure = (feature) -> new BigDecimal(measure.apply(reader.read(feature.getGeometry()))
.doubleValue());
}
public void run(Path extract, PrintStream out) throws IOException {
List<Feature> features = readFeatures(extract);
System.out.println(Iterables.size(features));
features.sort((a, b) -> validFrom(a).compareTo(validFrom(b)));
print(features, out);
}
private void print(List<Feature> events, PrintStream out) {
LocalDateTime k = LocalDateTime.parse("2010-01-01T00:00Z", timeFormatter).truncatedTo(ChronoUnit.HOURS);
LocalDateTime kPlus1 = k.plusHours(1);
PeekingIterator<Feature> itr = peekingIterator(events.iterator());
PriorityQueue<Feature> active = new PriorityQueue<>((a, b) -> validTo(a).compareTo(validTo(b)));
printHeader(out);
while (itr.hasNext()) {
modCreation = modDeletion = modTagChange = modGeomChange = modTagGeomChange = 0;
while (itr.hasNext() && validTo(itr.peek()).isBefore(k)) {
itr.next();
}
removeNonActive(active, kPlus1);
while (itr.hasNext() && validFrom(itr.peek()).isBefore(kPlus1) && validTo(itr.peek()).isAfter(k)) {
Feature feature = itr.next();
if (validTo(feature).compareTo(kPlus1) <= 0) {
if (feature.getProperties()
.containsKey("@creation")) {
modCreation++;
} else if (feature.getProperties()
.containsKey("@tag-change")) {
if (feature.getProperties()
.containsKey("@geometry-change")) {
modTagGeomChange++;
} else {
modTagChange++;
}
} else if (feature.getProperties()
.containsKey("@geometry-change")) {
modGeomChange++;
}
continue;
}
add(active, feature);
}
if (k.compareTo(startState) >= 0) {
print(k, kPlus1, out);
}
k = kPlus1;
kPlus1 = k.plusHours(1);
}
while (!active.isEmpty()) {
modCreation = modDeletion = modTagChange = modGeomChange = modTagGeomChange = 0;
removeNonActive(active, kPlus1);
print(k, kPlus1, out);
k = kPlus1;
kPlus1 = k.plusHours(1);
if (k.compareTo(endState) >= 0)
break;
}
}
private List<Feature> readFeatures(Path path) throws IOException {
String json = new String(Files.readAllBytes(path));
FeatureCollection featureCollection = (FeatureCollection) GeoJSONFactory.create(json);
return getEvents(Arrays.asList(featureCollection.getFeatures()));
}
private List<Feature> getEvents(List<Feature> features) {
List<Feature> events = new ArrayList<>();
final PeekingIterator<Feature> upstream = peekingIterator(features.iterator());
while (upstream.hasNext()) {
Feature feature = upstream.peek();
OSMId osmId = osmId(feature);
while (upstream.hasNext() && osmId(upstream.peek()).equals(osmId)) {
feature = upstream.next();
if (validFrom(feature).isBefore(endState)) {
events.add(feature);
}
}
}
return events;
}
private void removeNonActive(PriorityQueue<Feature> active, LocalDateTime k) {
while (!active.isEmpty() && validTo(active.peek()).compareTo(k) < 0) {
Feature feature = active.poll();
OSMId osmId = osmId(feature);
if (feature.getProperties()
.containsKey("@deletion")) {
modDeletion++;
}
BigDecimal value = measure.apply(feature);
boolean inFinalStage =
feature.getProperties().containsKey("@stage-3") &&
feature.getProperties().containsKey("@stage-after");
if (inFinalStage) {
numBuildingsInFinalStage -= 1;
measureInFinalStage = measureInFinalStage.subtract(value);
// double distance = DiscreteHausdorffDistance.distance(geom(feature), geom(endStage));
// distanceToFinalStage = distanceToFinalStage.subtract(new BigDecimal(distance));
} else {
wrongBuildings -= 1;
wrongMeasure = wrongMeasure.subtract(value);
}
}
}
private void add(PriorityQueue<Feature> active, Feature feature) {
active.add(feature);
if (feature.getProperties()
.containsKey("@creation")) {
modCreation++;
} else if (feature.getProperties()
.containsKey("@tag-change")) {
if (feature.getProperties()
.containsKey("@geometry-change")) {
modTagGeomChange++;
} else {
modTagChange++;
}
} else if (feature.getProperties()
.containsKey("@geometry-change")) {
modGeomChange++;
}
BigDecimal value = measure.apply(feature);
//Feature endStage = endStageFeatures.get(osmId(feature));
boolean inFinalStage =
feature.getProperties().containsKey("@stage-3") &&
feature.getProperties().containsKey("@stage-after");
if (inFinalStage) {
OSMId osmId = osmId(feature);
numBuildingsInFinalStage += 1;
measureInFinalStage = measureInFinalStage.add(value);
//double distance = DiscreteHausdorffDistance.distance(geom(feature), geom(endStage));
//distanceToFinalStage = distanceToFinalStage.add(new BigDecimal(distance));
} else {
wrongBuildings += 1;
wrongMeasure = wrongMeasure.add(value);
}
}
private Geometry geom(Feature feature) {
return reader.read(feature.getGeometry());
}
private void printHeader(PrintStream out) {
out.println("k, k_plus1" + ", final_stage" //
+ ", wrong" //
+ ", final_stage_measure" //
+ ", wrong_measure" //
+ ", distance" //
+ ", total_num_objects" //
+ ", modCreation" //
+ ", modDeletion" //
+ ", modTagChange" //
+ ", modGeomChange" //
+ ", modTagGeomChange");
}
private void print(LocalDateTime k, LocalDateTime kPlus1, PrintStream out) {
out.println(k + "," + kPlus1 //
+ "," + numBuildingsInFinalStage //
+ "," + wrongBuildings //
+ "," + measureInFinalStage.doubleValue() //
+ "," + wrongMeasure.doubleValue() //
+ "," + distanceToFinalStage //
+ "," + (numBuildingsInFinalStage + wrongBuildings) + "," + modCreation //
+ "," + modDeletion //
+ "," + modTagChange //
+ "," + modGeomChange //
+ "," + modTagGeomChange);
}
private static class OSMId {
public final OSMType type;
public final long id;
public OSMId(OSMType type, long id) {
this.type = type;
this.id = id;
}
@Override
public int hashCode() {
return Objects.hash(id, type);
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (!(obj instanceof OSMId))
return false;
OSMId other = (OSMId) obj;
return id == other.id && type == other.type;
}
}
private static OSMId osmId(Feature f) {
OSMType type = OSMType.valueOf((String) f.getProperties()
.get("@osmType"));
Object obj = f.getProperties()
.get("@osmId");
long id;
if (obj instanceof Integer) {
id = ((Integer) obj).intValue();
} else if (obj instanceof Long) {
id = ((Long) obj).longValue();
} else {
throw new RuntimeException("could not convert osmId! (" + obj + ")");
}
return new OSMId(type, id);
}
private static LocalDateTime validFrom(Feature f) {
return LocalDateTime.parse((String) f.getProperties()
.get("@validFrom"), timeFormatter);
}
private static LocalDateTime validTo(Feature f) {
return LocalDateTime.parse((String) f.getProperties()
.get("@validTo"), timeFormatter);
}
}
package org.heigit.ohsome.oshdb.gyrt;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import org.heigit.bigspatialdata.oshdb.util.time.OSHDBTimestamps;
public class Config {
public static final List<Slum> SLUMS = Slum.ALL;
public static final List<String> KEYS = List.of("building", "highway");
public static final String isoDateStart = "2010-01-01T00:00:00Z";
public static final String isoDateEnd = "2020-01-01T00:00:00Z";
public static final OSHDBTimestamps fullHistory= new OSHDBTimestamps(isoDateStart, isoDateEnd);
public static final Path geojsons = Paths.get("sites");
public static final Path igniteCfg = Paths.get("PATH_TO_IGNITE_CONFIGURATION");
public static final String prefix = "PREFIX";
public static final String keytableUrl = "URL_TO_KEYTABLE_DATABASE";
}
package org.heigit.ohsome.oshdb.gyrt;
import static org.heigit.ohsome.oshdb.gyrt.util.OSHDBUtil.connectToIgnite;
import static org.heigit.ohsome.oshdb.gyrt.util.OSHDBUtil.loadGeoJSON;
import java.io.PrintWriter;
import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.heigit.bigspatialdata.oshdb.api.generic.function.SerializableFunction;
import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
import org.heigit.bigspatialdata.oshdb.osm.OSMEntity;
import org.heigit.bigspatialdata.oshdb.osm.OSMType;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTagKey;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
import org.heigit.bigspatialdata.oshdb.util.taginterpreter.DefaultTagInterpreter;
import org.heigit.bigspatialdata.oshdb.util.taginterpreter.TagInterpreter;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.OSMTag;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
import org.heigit.ohsome.oshdb.gyrt.util.ThrowingConsumer;
import org.heigit.ohsome.oshdb.gyrt.util.OSHDBUtil.OSHDBWrapper;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.Polygonal;
import org.locationtech.jts.geom.PrecisionModel;
import org.locationtech.jts.precision.GeometryPrecisionReducer;
import org.wololo.geojson.Feature;
import org.wololo.jts2geojson.GeoJSONWriter;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
import static org.heigit.ohsome.oshdb.gyrt.Slum.*;
public class Extract {
public static void main(String[] args) throws Exception {
Path data = Paths.get("/home/rtroilo/godwin-gyrt/paper/data");
extract(data);
}
public static void extract(Path outputPath) throws Exception {
Files.createDirectories(outputPath);
System.out.println("extracting >>>");
Stopwatch stopwatch = Stopwatch.createStarted();
connectToIgnite(Config.igniteCfg, Config.prefix, Config.keytableUrl, run(outputPath));
System.out.println("<<< " + stopwatch);
}
public static ThrowingConsumer<OSHDBWrapper, Exception> run(Path outputDir) {
return (oshdb) -> {
TagTranslator tagTranslator = oshdb.getDefaultTagTranslator();
TagInterpreter tagInterpreter = new DefaultTagInterpreter(tagTranslator);
Map<String, OSHDBTagKey> map = new HashMap<>();
for(String key : Config.KEYS) {
map.put(key, tagTranslator.getOSHDBTagKeyOf(key));
}
List<Slum> slums = Config.SLUMS;
for (Slum slum : slums) {
Geometry site = loadGeoJSON(Config.geojsons.resolve(slum.name + ".geojson"))[0];
for (Map.Entry<String, OSHDBTagKey> key : map.entrySet()) {
Path outputFile = outputDir.resolve(slum.name +"_"+key.getKey().toString()+"_"+Config.isoDateStart+"_"+Config.isoDateEnd+".geojson");
try (PrintWriter output = new PrintWriter(Files.newBufferedWriter(outputFile))) {
System.out.println(outputFile);
List<Feature> features = oshdb.getContributionView() //
.areaOfInterest((Geometry & Polygonal) site) //
.timestamps(Config.fullHistory) //
.osmTag(key.getKey()) //
.groupByEntity() //
.map(Extractor.map(slum, key.getValue(), tagInterpreter)) //
.flatMap(f -> f)
.collect();
features.forEach(feature -> {
Map<String, Object> props = feature.getProperties();
OSHDBTag[] tags = (OSHDBTag[]) props.remove("@tags");
for (OSHDBTag tag : tags) {
OSMTag osmTag = tagTranslator.getOSMTagOf(tag);
props.put(osmTag.getKey(), osmTag.getValue());
}
});
output.print("{");
output.print(
"\"attribution\" : { \"url\" : \"https://ohsome.org/copyrights\",\"text\" : \"© OpenStreetMap contributors\"},");
output.print("\"apiVersion\" : \"1.0\",");
output.print("\"type\" : \"FeatureCollection\",");
output.println("\"features\" : [");
Iterator<Feature> itr = features.iterator();
while (itr.hasNext()) {
output.print(itr.next());
if (itr.hasNext()) {
output.println(",");
}
}
output.println();
output.print("]");
output.println("}");
}
}
}
};
}
public static class Extractor implements Serializable {
private static final long serialVersionUID = 2L;
private final Slum slum;
private final OSHDBTagKey key;
private final GeoJSONWriter writer = new GeoJSONWriter();
private final PrecisionModel pm = new PrecisionModel(10000000.0);
private final GeometryPrecisionReducer precisionReducer = new GeometryPrecisionReducer(pm);
private Extractor(Slum slum, OSHDBTagKey key, TagInterpreter tagInterpreter) {
this.slum = slum;
this.key = key;
}
public static SerializableFunction<List<OSMContribution>, List<Feature>> map(Slum slum, OSHDBTagKey key,
TagInterpreter tagInterpreter) {
Extractor extractor = new Extractor(slum, key, tagInterpreter);
return extractor::map;
}
public List<Feature> map(List<OSMContribution> contribs) {
List<Feature> features = new ArrayList<>(contribs.size());