Commit b3987d91 authored by Moritz Schott's avatar Moritz Schott
Browse files

clean up examples

parent 1ed328fc
package org.heigit.bigspatialdata.oshdb.examples;
import static java.lang.Math.max;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.Polygonal;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedMap;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.heigit.bigspatialdata.oshdb.api.db.OSHDBH2;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.OSMContributionView;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.OSMEntitySnapshotView;
import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
import org.heigit.bigspatialdata.oshdb.osm.OSMRelation;
import org.heigit.bigspatialdata.oshdb.osm.OSMType;
import org.heigit.bigspatialdata.oshdb.osm.OSMWay;
import org.heigit.bigspatialdata.oshdb.util.OSHDBBoundingBox;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTagKey;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
import org.heigit.bigspatialdata.oshdb.util.time.OSHDBTimestamps.Interval;
import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
import similarProjectsComparison.oshdb.geomdistance.Junctions.WayNode;
public class EquidistanceDevelopment{
public static void main(String[] args) throws Exception {
// database
OSHDBH2 oshdb = (new OSHDBH2(
"C:\\Users\\ob437\\eclipse-workspace\\oshdb-examples\\oshdb-examples\\workshops\\src\\main\\resources\\bw_z15_keytables.compressed.oshdb.mv.db"))
.multithreading(!true);
// OSHDBH2 oshdbKeytables = new OSHDBH2("C:\\Users\\ob437\\eclipse-workspace\\oshdb-examples\\oshdb-examples\\workshops\\src\\main\\resources\\bw_z15_keytables.compressed.oshdb.mv.db");
OSHDBH2 oshdbKeytables = oshdb;
TagTranslator tt = new TagTranslator(oshdbKeytables.getConnection());
OSHDBTag typeMultipolyTag = tt.getOSHDBTagOf("type", "multipolygon");
OSHDBTagKey naturalKey = tt.getOSHDBTagKeyOf("natural");
OSHDBTagKey landuseKey = tt.getOSHDBTagKeyOf("landuse");
// query
List<ImmutablePair<Double, Double>> devel_equ = OSMContributionView.on(oshdb)
.keytables(oshdbKeytables)
//.areaOfInterest(new OSHDBBoundingBox(8.6598,49.3966,8.726,49.4249)) // HD
.areaOfInterest(new OSHDBBoundingBox(8.9079,48.6626,9.4469,48.8959)) // Stuttgart
/*.areaOfInterest(new OSHDBBoundingBox(80.83053588867188,
26.722307016045065, 81.1065673828125,
26.96124577052697)) // lucknow indien*/
/*.areaOfInterest(new OSHDBBoundingBox(72.762451171875,
18.884198609350264, 72.99728393554688,
19.32539900916396)) // mumbai indien*/
/*.areaOfInterest(new OSHDBBoundingBox(8.631992340087889,
49.38226103399081, 8.732757568359375,
49.44212421583237)) // hd barron*/
.timestamps("2008-01-01", "2017-01-01")
.osmType(OSMType.WAY, OSMType.RELATION)
.osmEntityFilter(osmEntity -> {
// get relation only if it is a multipolygon
if (osmEntity.getType() == OSMType.RELATION) {
if (!osmEntity.hasTagValue(typeMultipolyTag.getKey(), typeMultipolyTag.getValue())) {
return false;
}
}
// get only ways and relations with natural or landuse tag
return osmEntity.hasTagKey(naturalKey) || osmEntity.hasTagKey(landuseKey);
})
.groupByEntity()
.filter(osmContributions -> {
// exclude objects where area change is "large" or which are not polygons
//deletion (getGeometryBefore) or not deletion (getGeometryAfter)
OSMContribution lastContribution = osmContributions.get(osmContributions.size() - 1);
Geometry geom_first = osmContributions.get(0).getGeometryUnclippedAfter();
Geometry geom_last;
if (lastContribution.is(ContributionType.DELETION)) {
geom_last = lastContribution.getGeometryUnclippedBefore();
} else {
geom_last = lastContribution.getGeometryUnclippedAfter();
}
if (!(geom_first instanceof Polygonal && geom_last instanceof Polygonal))
return false; // is not a polygon
double area_first = geom_first.getArea();
double area_last = geom_last.getArea();
if (Math.min(area_first, area_last) / Math.max(area_first, area_last) < 0.5)
return false; // area change is to large
return true;
})
.map(osmContributions -> {
//creation -> use getGeometryAfter
Geometry geometry_first = osmContributions.get(0).getGeometryUnclippedAfter();
double equidistance_first =
Geo.lengthOf(geometry_first.getBoundary()) / geometry_first.getCoordinates().length;
//deletion (getGeometryBefore) or nicht deletion (getGeometryAfter)
OSMContribution lastContribution = osmContributions.get(osmContributions.size() - 1);
Geometry geometry_last;
if (lastContribution.is(ContributionType.DELETION)){
geometry_last = lastContribution.getGeometryUnclippedBefore();
}else {
geometry_last = lastContribution.getGeometryUnclippedAfter();
}
double equidistance_last =
Geo.lengthOf(geometry_last.getBoundary()) / geometry_last.getCoordinates().length;
// Development of the equidistance: Substract equidistance from polygon
// which is recently valid from the initially created corresponding one
// equidistance_first - equidistance_last
return new ImmutablePair<>(equidistance_first, equidistance_last);
})
.collect();
// sort by equidistance_first
devel_equ.sort(Comparator.comparingDouble(Pair::getLeft));
// output
for (ImmutablePair<Double, Double> result : devel_equ)
System.out.format("%.2f\t;%.2f\n", result.getLeft(), result.getRight());
// write result to txt file
BufferedWriter writer = null;
try {
//create a temporary file
String fileName = "equ_stg.txt";
File logFile = new File(fileName);
writer = new BufferedWriter(new FileWriter(logFile));
for (ImmutablePair<Double, Double> result : devel_equ) {
String line = result.getLeft().toString() + " ; " + result.getRight().toString();
writer.write(line);
writer.newLine();
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
// Close the writer regardless of what happens...
writer.close();
} catch (Exception e) {
}
}
}
}
package org.heigit.bigspatialdata.oshdb.examples;
import com.google.common.base.Functions;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.commons.lang3.time.StopWatch;
import org.heigit.bigspatialdata.oshdb.api.db.OSHDBH2;
import org.heigit.bigspatialdata.oshdb.api.generic.OSHDBCombinedIndex;
import org.heigit.bigspatialdata.oshdb.api.mapreducer.OSMEntitySnapshotView;
import org.heigit.bigspatialdata.oshdb.osm.OSMType;
import org.heigit.bigspatialdata.oshdb.osm.OSMWay;
import org.heigit.bigspatialdata.oshdb.util.OSHDBBoundingBox;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTag;
import org.heigit.bigspatialdata.oshdb.util.OSHDBTimestamp;
import org.heigit.bigspatialdata.oshdb.util.geometry.Geo;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.OSMTag;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
import org.heigit.bigspatialdata.oshdb.util.time.OSHDBTimestamps;
public class Junctions {
public static class WayNode {
long id;
OSHDBTag streetName;
Coordinate coord;
public WayNode(long id, OSHDBTag oshdbTag, Coordinate coordinate) {
this.id = id;
this.streetName = oshdbTag;
this.coord = coordinate;
}
}
public static double getAngle(double x1,double y1, double x2, double y2 ) {
//double angle = (float) Math.toDegrees(Math.atan2(targetPt.getCentroid().getY() - centerPt.getCentroid().getY(), targetPt.getCentroid().getX() - centerPt.getCentroid().getX()));
double angle = (float) Math.toDegrees(Math.atan2(y2 - y1, x2 - x1));
if(angle < 0){
angle += 360;
}
return angle;
}
static int c;
public static void main(String[] args) throws Exception {
StopWatch s = new StopWatch(); s.start();
OSHDBH2 oshdb = (new OSHDBH2("C:\\Users\\ob437\\eclipse-workspace\\oshdb-examples\\oshdb-examples\\workshops\\src\\main\\resources\\india_z15_keytables.compressed.oshdb.mv.db")).multithreading(!true);
OSHDBH2 oshdbKeytables = oshdb;
TagTranslator tt = new TagTranslator(oshdbKeytables.getConnection());
int nameKey = tt.getOSHDBTagKeyOf("name").toInt();
OSHDBTimestamps timestamps = new OSHDBTimestamps("2014-06-06", "2018-07-06");
SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, Long>, List<WayNode>> results = OSMEntitySnapshotView
.on(oshdb)
.keytables(oshdbKeytables)
//.areaOfInterest(new OSHDBBoundingBox(8.9079,48.6626,9.4469,48.8959)) // Stuttgart
//.areaOfInterest(new OSHDBBoundingBox(9.1351,48.7555,9.2527,48.8260)) // Stuttgart klein
/*.areaOfInterest(new OSHDBBoundingBox(80.83053588867188,
26.722307016045065,81.1065673828125,
26.96124577052697)) // lucknow indien*/
.areaOfInterest(new OSHDBBoundingBox(72.762451171875,
18.884198609350264, 72.99728393554688,
19.32539900916396)) // mumbai indien
/*.areaOfInterest(new OSHDBBoundingBox(8.631992340087889,
49.38226103399081, 8.732757568359375,
49.44212421583237)) // hd barron*/
//.areaOfInterest(new OSHDBBoundingBox( 8.69004,49.3991,8.6979,49.40541))
.timestamps(timestamps)
.osmTag("highway")
.osmTag("name")
.osmType(OSMType.WAY)
.aggregateByTimestamp()
.flatMap(snapshot -> {
OSMWay way = (OSMWay) snapshot.getEntity();
LinkedList<WayNode> ret = new LinkedList<>();
way.getRefEntities(snapshot.getTimestamp()).forEach(node -> {
for (OSHDBTag tag : way.getTags()) {
if (tag.getKey() == nameKey) {
ret.add(new WayNode(
node.getId(),
tag,
new Coordinate(node.getLongitude(), node.getLatitude())
));
}
}
});
return ret;
})
.aggregateBy(wayNode -> wayNode.id)
.collect();
Map<String, TreeMap<OSHDBTimestamp, Coordinate>> filteredResults = results
.entrySet().stream().filter(result -> {
final List<WayNode> wayNodeList = result.getValue();
if (wayNodeList.size() <= 1) {
return false;
}
if (wayNodeList.stream()
.collect(Collectors.groupingBy(wayNode -> wayNode.streetName, Collectors.counting()))
.size() == 1) {
return false;
}
return true;
}).collect(Collectors.toMap(
result -> {
return result.getValue().stream().map(wayNode -> wayNode.streetName)
.collect(Collectors.groupingBy(Functions.identity(), Collectors.counting()))
.keySet().stream()
.map(tt::getOSMTagOf)
.map(OSMTag::getValue)
.sorted()
.reduce((a, b) -> a + "-" + b).orElse("?");
},
result -> {
TreeMap<OSHDBTimestamp, Coordinate> ret = new TreeMap<>();
ret.put(result.getKey().getFirstIndex(), result.getValue().get(0).coord);
return ret;
},
(map1, map2) -> {
TreeMap<OSHDBTimestamp, Coordinate> ret = new TreeMap<>();
ret.putAll(map1);
ret.putAll(map2);
return ret;
}
));
// to make a diagram, save txt and use it in polarplot.py
FileWriter f0 = new FileWriter("your\\directory\\mum20_junct.txt");
String newLine = System.getProperty("line.separator");
filteredResults.forEach((junctionName, value) -> {
if (value.size() <= 1) {
System.out.println("only present at < two timestamps");
} else {
c += 1;
double g = (Geo.distanceBetweenCoordinates(
value.firstEntry().getValue().y,
value.firstEntry().getValue().x,
value.lastEntry().getValue().y,
value.lastEntry().getValue().x
));
if (g < 20.000){
System.out.print(junctionName + "; ");
System.out.print(Geo.distanceBetweenCoordinates(
value.firstEntry().getValue().y,
value.firstEntry().getValue().x,
value.lastEntry().getValue().y,
value.lastEntry().getValue().x
));
System.out.println(" ; " + getAngle(
value.firstEntry().getValue().x,
value.firstEntry().getValue().y,
value.lastEntry().getValue().x,
value.lastEntry().getValue().y
));
String text = Geo.distanceBetweenCoordinates(
value.firstEntry().getValue().y,
value.firstEntry().getValue().x,
value.lastEntry().getValue().y,
value.lastEntry().getValue().x
)+" ; " + getAngle(
value.firstEntry().getValue().x,
value.firstEntry().getValue().y,
value.lastEntry().getValue().x,
value.lastEntry().getValue().y
);
try {
f0.write(text + newLine);
} catch (IOException e) {
e.printStackTrace();
}
}
}
});
f0.close();
s.stop();
System.out.println(s);
System.out.println(c);
}
}
## Analysis of OSM data: road length, positional accuracy of the data, number of contributors and equidistance development of "landuse" and "natural" polygons
In "A Comprehensive Framework for Intrinsic OpenStreetMap Quality Analysis" (2014)
Barron, Neis and Zipf analyzed road length, positional accuracy, number of contributors and
changes in landuse-polygons' geometry in OSM history data of three cities with the framework iOSMAnalyzer.
This project repeats this analysis but with other similar frameworks/projects.
The first framework is the oshdb API.
The analysis is made with data of four cities,
Heidelberg (8.631992340087889, 49.38226103399081, 8.732757568359375, 49.44212421583237),
Stuttgart (8.9079, 48.6626, 9.4469, 48.8959),
Mumbai (72.762451171875, 18.884198609350264, 72.99728393554688, 19.32539900916396)
and Lucknow (80.83053588867188, 26.722307016045065,81.1065673828125, 26.96124577052697).
**RoadLength**
Calculates the length of different road types to get an overview about the road network completeness.
OSM Ways are considered having the key-tag highway and a certain group of values.
The different value groups are:
- highway-motorway:
"motorway","primary","motorway_link","primary_link","trunk","trunk_link"
- secondary-tertiary:
"secondary","secondary_link","tertiary","tertiary_link"
- residential:
"residential","living_street"
- other:
"unclassified","road","service","track","path","pedestrian","footway","cycleway","steps",
"platform","bridleway"
The output shows for every timestamp the road length in meters.
**Junctions**
Analysis of the distance and angle between actual positions of road junctions and their previous
positions to find out the accuracy of the data.
Approach similar to [this suggestion]
(https://stackoverflow.com/questions/12965090/get-list-of-all-intersections-in-a-city/18381500#18381500).
The output shows for every timestamp the distance in meters and the angle in degree.
To plot the result, you could copy paste the output in a .txt file and use it in polarplot.py.
**DistinctContributors**
Calculates the amount of distinct contributors per timestamp.
This information is important to be able to say something about the road network completeness.
According to Barron, Neis, Zipf (2014) if only few active contributors are detected for a city,
probably an external data source is needed to state something about the road completeness.
The output shows for every timestamp the number of distinct contributors.
**EquidistanceDevelopment**
Calculates the equisdistance of polygons with "landuse" or "natural" tag. Therefore the length of the
boundary of the polygon is divided by the number of points belonging to the polygon.
*"The lower the equidistance of the currently
valid polygon compared with its initially created version, the better the polygon’s relative
quality development, due to further editing which potentially led to a more precise geometric
representation."* (Barron, Neis, Zipf, 2014, p. 888)
To get the equidistance development, the euqidistance of the current version is substracted from the euqidistance of the first version of the corresponding polygon.
In the diagram made with equidistancePlot.py the x-axis shows the number of polygons and the y-axis shows the equidistance development.
The polygons are sorted by the equidistance in their first version in ascending order.
City | Improvement of equidistance average | Unchanged polygons | per cent of polygons with improvemnt of geometric representation
Stuttgart | 4,2 m | 59,70 % | 29,5 %
Heidelberg | 4 m | 60,90 % | 26,6 %
Mumbai | 2,6 m | 69,40 % | 21,2 %
Lucknow | 0, 84 m | 93,40 % | 5,3 %
To plot the result, you could use the created .txt file in equidistancePlot.py.
# -*- coding: utf-8 -*-
#!/usr/bin/python2.7
import psycopg2
from pylab import *
import matplotlib
import psycopg2
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import numpy as np
import sys
def equidistance(inTextFile,outJpegFile):
equ_first_list = []
equ_last_list = []
equ_dev_list = []
with open(inTextFile) as f:
for line in f.readlines():
line = line.split(";")
if len(line) > 0:
equ_first = line[0].replace(",",".")
equ_first = float(equ_first)
equ_first_list.append(equ_first)
equ_last = line[1].replace(",",".")
equ_last = float(equ_last)
equ_last_list.append(equ_last)
equ_dev_list.append(equ_first-equ_last)
# improvement of equidistance average
equ_av = sum(equ_dev_list)/len(equ_dev_list)
# not changed polygons
poly_unchanged = (100.0/len(equ_dev_list))* (equ_dev_list.count(0.0))
# improvement of geometric representation of polygons
count_improved = 0
for i, equ_first in enumerate(equ_first_list):
if equ_last_list[i] < equ_first:
count_improved += 1
improved_geom = (100.0/len(equ_first_list)) * count_improved
print str(count_improved) + " " + str(len(equ_first_list))
print "improvement of equidistance average " + str(equ_av) +" m"
print "per cent of unchanged polygons " + str(poly_unchanged) + " %"
print "per cent of polygons with improvemnt of geometric representation " + str(improved_geom) + " %"
###
### Plot (Line-Chart)
###
fig, ax = plt.subplots()
# Create linechart
plt.plot(equ_dev_list, color = '#ff6700', linewidth=2, label='Development of the Equidistance')
# Place a gray dashed grid behind the thicks (only for y-axis)
ax.xaxis.grid(color='gray', linestyle='dashed')
ax.yaxis.grid(color='gray', linestyle='dashed')
# Set this grid behind the thicks
ax.set_axisbelow(True)
# Rotate x-labels on the x-axis
fig.autofmt_xdate()
# Label x and y axis
x_str = ('Polygons with a "natural" or "landuse"-Tag')
plt.xlabel(x_str)
plt.ylabel('Change of Equidistance [m]')
# place legend
ax.legend(loc='upper center', prop={'size':12})
# Plot-title
plt.title('Equidistance Development of Polygons with "natural" or "landuse"-Tag"')
# Save plot to *.jpeg-file
plt.savefig(outJpegFile)
plt.clf()
return
if __name__ == '__main__':
if len(sys.argv) == 3:
inTextFile = sys.argv[1]
outJpegFile = sys.argv[2]
equidistance(inTextFile,outJpegFile)
else:
print "usage: 3 commandline arguments are needed: the script itself, the .txt file \nwith the equidistance results and a .jpeg filepath to save the plot.\n<script.py inTextfile.txt outJpegFile.jpeg>"
\ No newline at end of file
# -*- coding: utf-8 -*-
#!/usr/bin/python2.7
import psycopg2
from pylab import *
import matplotlib
import psycopg2
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import numpy as np
dist_lilst = []
angle_list = []
with open("your\\directory\\mum20_junct.txt") as f:
for line in f.readlines():
line = line.split(";")
dist = line[0]
dist = float(dist)
dist_lilst.append(dist)
angle = line[1]
angle = float(angle)
angle_list.append(angle)
col1 = dist_lilst# Distance between two junctions
col2 = angle_list# Degree of Distance
###
### Plot (Scatter-Plot)
###
ax = subplot(111, polar=True)
# Scatterplot: degree in Polar-Scatter (degree between different junctions), y-axis in Scatter (length betweeen different junctions), color, size of circles, color-map:
c = scatter(col2, col1, c=col1, s=100, cmap=cm.hsv)
# Set Opacity of circles
c.set_alpha(0.75)
# Plot-title
plt.title("Directional Scatterplot of Degree and Distance [m] between two Junctions")
# Save plot to *.png-file
plt.savefig('your\\directory\\junctions_luc.jpeg')
plt.clf()
\ No newline at end of file
{
"osmatrix-db": {"connection":"jdbc:postgresql://lemberg.geog.uni-heidelberg.de:5432/osmatrixhd", "user":"osmatrix", "password":"osmatrix2016"},
"osh-db": {"connection":"jdbc:h2:./heidelberg--2017-05-29", "user":"sa", "password":""},
"temp-db": {"connection": "jdbc:postgresql://lemberg.geog.uni-heidelberg.de:5432/osmatrixhd", "user":"osmatrix", "password":"osmatrix2016"},
"timestamps": {"start": "2012-01","end":"2012-12", "steps":"PM1"},
"bbox":[8.56982,8.79541,49.3504,49.4616],