Skip to content

GitLab

Commit 26f4bc9e authored by Lukas Loos's avatar Lukas Loos
Browse files

initial commit: moved ohsome2x-cli in ohsome2x

parent cfee8b85
Hide whitespace changes
Inline Side-by-side
FeatureType.js 0 → 100644
View file @ 26f4bc9e
// const PgAsync = require('pg-async').default;
//const SQL = require('pg-async').SQL;
// const axios = require('axios');
// const querystring = require('querystring');
// const turfHelpers = require('@turf/helpers');
// const PgFeatureType = require('./PgFeatureType.js');
//abstract class and factory for postgis, GeoJSON, ...
class FeatureType {
constructor(config) {
this.config = config;
}
/**
* @returns GeoJSON<FeatureCollection>
**/
async getFeatures(){}
async writeFeatures(featureCollection){}
async writeOhsomeFeatures(ohsomeGroupByBoundaryResponseJSON, horizontalTimestampColumns){}
/**
* physically deletes the feature type from the store (e.g. file from disk or table from database)
**/
async delete(){}
async finalize(){}
}
module.exports = FeatureType;
FeatureTypeFactory.js 0 → 100644
View file @ 26f4bc9e
const PgFeatureType = require('./PgFeatureType.js');
const GeoJsonFeatureType = require('./GeoJsonFeatureType.js');
const PgAsync = require('pg-async').default;
class FeatureTypeFactory {
//factory
static async create(featureTypeConfig){
const type = featureTypeConfig.store.type.toLowerCase();
switch (type) {
case 'postgis':
featureTypeConfig.schema = featureTypeConfig.schema || 'public';
try {
console.log(`Check if table ${featureTypeConfig.schema}.${featureTypeConfig.name} exists.`);
const tableExists = await this.checkTableExists(featureTypeConfig);
featureTypeConfig.exists = await tableExists;
return new PgFeatureType(featureTypeConfig);
} catch (e) {
console.log(e.toString());
throw new Error('Could not create PgFeatureType.');
}
break;
case 'geojson':
return new GeoJsonFeatureType(featureTypeConfig);
break;
default:
throw Error('No such store type available: ' + type);
}
}
static async checkTableExists(featureTypeConfig){
const sql = `SELECT EXISTS (
SELECT 1
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = '${featureTypeConfig.schema || 'public'}'
AND c.relname = '${featureTypeConfig.name}'
AND c.relkind = 'r' -- only tables
);`;
let db = new PgAsync(featureTypeConfig.store);
let exists;
try {
exists = await db.value(sql);
console.log(`Table ${featureTypeConfig.schema}.${featureTypeConfig.name} exists: ${exists}`);
return exists;
} catch (e) {
console.log(e);
throw new Error('Could not check if table exists. Is the database accessible?');
} finally {
db.closeConnections();
}
}
}
module.exports = FeatureTypeFactory;
GeoJsonFeatureType.js 0 → 100644
View file @ 26f4bc9e
const FeatureType = require('./FeatureType.js');
const fs = require('fs');
var path = require('path');
const turfHelpers = require('@turf/helpers');
const turfInvariant = require('@turf/invariant');
const Papa = require('papaparse');
const reproject = require('reproject').reproject;
const proj4 = require('proj4');
class GeoJsonFeatureType extends FeatureType {
constructor(config){
super(config);
this.store = config.store; //type:'geojson'
this.name = config.name || path.basename(config.store.path);
this.geometryId = config.geometryId || 'id';
this.path = config.store.path;
this.fileExists = fs.existsSync(this.path);
console.log('file exists: ' + this.fileExists);
}
delete(){
if(this.fileExists){
try {
console.log('DELETE GeoJSON file: ' + this.path);
fs.unlinkSync(this.path)
//file removed
} catch(err) {
console.error(err)
}
}
};
/**
* @returns GeoJSON<FeatureCollection>
**/
getFeatures(){
if(!fs.existsSync(this.path)){
throw new Error('Cannot get features. File does not exist: ' + this.path);
}
let geojson;
try {
let rawdata = fs.readFileSync(this.path);
geojson = JSON.parse(rawdata);
} catch (e) {
console.log(e);
}
try {
turfInvariant.geojsonType(geojson, 'FeatureCollection', 'getFeatures');
} catch (e) {
console.log(e);
throw new Error('GeoJSON must be of type FeatureCollection.');
}
//normalize id columnName
if(this.geometryId != 'id'){
geojson.features.forEach(
(feature)=>{
if(!feature.properties[this.geometryId]){
throw new Error(`No id column "${this.geometryId}" found on ${this.name}. Please configure geometryId: '<columnNameOfId>' in featureTypeConfig.`);
}
feature.properties.id = feature.properties[this.geometryId];
delete feature.properties[this.geometryId]
},this);
}
return geojson;
}
writeFeatures(featureCollection) {
//do nothing when feature collection is empty
if (featureCollection.features.length == 0) {
console.log('WARNING: No features in feature collection. No feature writing.');
return;
}
let data = JSON.stringify(featureCollection);
try {
console.log(`Saving GeoJSON file to: ${this.path}`);
fs.writeFileSync(this.path, data);
} catch (e) {
console.log(e);
}
}
static removeFeaturesByPropertyValue(featureCollection,propertyName, value){
let filteredFeatures = featureCollection.features.filter(
(feature)=>{
return feature.properties[propertyName] !== value;
}
);
featureCollection.features = filteredFeatures;
return featureCollection;
}
/**
* @param ohsomeGroupByBoundaryResponse response from ohsome-API, either ohsomeJSON or ohsomeGroupByBoundaryCSV
* @param horizontal boolean if the result featureType should have many or one timestamp column
* @param sourceFeatureCollection if not undefined, will be used to join input geometry (bpolys) and ohsome-API results
**/
static fromOhsome(ohsomeGroupByBoundaryResponse, horizontal, sourceFeatureCollection, transformToWebmercator) {
const ohsomeGroupByBoundaryCSV = {
delimiter: ';',
dynamicTyping: (col)=>(col != 0), //first column is timestamp and should remain string not auto converted to Date
header: false,
skipEmptyLines: true,
comments: '#',
newline: '\n'
};
const ohsomeResults = ohsomeGroupByBoundaryResponse;
const isCSV = (typeof ohsomeResults == 'string');
const shouldCreateGeometry = !!sourceFeatureCollection;
let targetFeatures = [];
let targetFeatureCollection;
let idGeomMap;
if(shouldCreateGeometry){
//create idGeomMap
idGeomMap = new Map();
if(transformToWebmercator){
console.log('TRANSFORMING...');
sourceFeatureCollection = reproject(sourceFeatureCollection, proj4.WGS84, proj4.defs['EPSG:3857'], proj4.defs);
}
sourceFeatureCollection.features.forEach((feature)=>idGeomMap.set(feature.properties.id, feature.geometry ));
}
if (!horizontal){
//vertical is the default if undefined
// ohsomeJSON
if(!isCSV){ohsomeResults.groupByResult.forEach((item)=>{
const id = item.groupByObject;
const geom = (shouldCreateGeometry)? idGeomMap.get(id) : null;
const idFeatures = item.result.map((tv)=>{
return turfHelpers.feature(geom,
{
id: id,
timestamp: tv.timestamp,
value: tv.value
})
});
targetFeatures.push(...idFeatures);
});}
if(isCSV){
//csv
let ohsomeCsv = Papa.parse(ohsomeResults,ohsomeGroupByBoundaryCSV).data;
let nrows = ohsomeCsv.length;
let ncols = ohsomeCsv[0].length;
for (let col = 1; col < ncols; col++) {
for (let row = 1; row < nrows; row++) {
let properties = {};
properties.timestamp = ohsomeCsv[row][0];
properties.id = ohsomeCsv[0][col];
properties.value = ohsomeCsv[row][col];
const geom = (shouldCreateGeometry)? idGeomMap.get(properties.id) : null;
targetFeatures.push(turfHelpers.feature(geom,properties));
}
}
}
} else {
//horizontal
if(!isCSV){
targetFeatures = ohsomeResults.groupByResult.map((item)=>{
const id = item.groupByObject;
const geom = (shouldCreateGeometry)? idGeomMap.get(id) : null;
let properties = {id: id};
item.result.forEach((tv)=>{properties[tv.timestamp] = tv.value});
return turfHelpers.feature( geom, properties);
});
}
if(isCSV){
//csv
let ohsomeCsv = Papa.parse(ohsomeResults,ohsomeGroupByBoundaryCSV).data;
let nrows = ohsomeCsv.length;
let ncols = (!!ohsomeCsv[0])? ohsomeCsv[0].length : 0;
for (let col = 1; col < ncols; col++) {
let properties = {};
properties.id = ohsomeCsv[0][col];
for (let row = 1; row < nrows; row++) {
properties[ohsomeCsv[row][0]] = ohsomeCsv[row][col]; // timestamp = value
}
const geom = (shouldCreateGeometry)? idGeomMap.get(properties.id) : null;
targetFeatures.push(turfHelpers.feature(geom,properties));
}
}
}
targetFeatureCollection = turfHelpers.featureCollection(targetFeatures);
if (shouldCreateGeometry && transformToWebmercator){
targetFeatureCollection.crs = {"type":"name","properties":{"name":"EPSG:3857"}};
}
return targetFeatureCollection;
}
}
module.exports = GeoJsonFeatureType;
Ohsome2X.js 0 → 100644
View file @ 26f4bc9e
// const FeatureType = require('./FeatureType.js');
const PgFeatureType = require('./PgFeatureType.js');
const querystring = require('querystring');
const turfHelpers = require('@turf/helpers');
const turfArea = require('@turf/area');
const axios = require('axios');
const normalizeUrl = require('normalize-url');
const FeatureTypeFactory = require('./FeatureTypeFactory.js');
const GeoJsonFeatureType = require('./GeoJsonFeatureType.js');
const defaultConfig = require('./conf/default.js');
let OHSOME_API_URL = normalizeUrl(defaultConfig.OHSOME_API_URL); //remove trailing slash and other things
class Ohsome2X {
constructor(config){
// let ohsomeQuery = {
// queryType: 'elements/count',
// keys: ['natural'].join(),
// values: ['tree'].join(),
// types: ['node'],
// time: '2014-01-01/2017-01-01/P1Y'
// }
//
// let store = {
// type: 'postgis', //postgis, TODO: geoJSON
// host: 'example.com',
// port: 5432,
// user: 'foo',
// password: '*****',
// database: 'your-db-name',
// //schema:
// }
//
// let sourceFeatureType = {
// name:'isea3h_nepal_res10_verkl', //postgis: tableName, geojson: optional name
// geometryId: 'id', //postgisOnly //unique not null, will be sorted when using fetchSize
// geometryColumn: 'geom', //postgisOnly //must be in EPSG:4326
// store: store, //store eg. postgis, geoJSON<FeatureCollection>,...
// fetchSize: 0 //postgisOnly
// }
//
// let targetFeatureType = {
// name: ohsomeQuery.keys[0] + '_' +ohsomeQuery.values[0],
// store: store,
// horizontalTimestampColumns: false,
// createGeometry: false, //TODO default should be true
// }
//
//
// //let
// config = {
// source: sourceFeatureType,
// target: targetFeatureType,
// ohsomeQuery: ohsomeQuery
// }
this.config = config;
this.cursor = this.config.source.cursor || 0;
this.fetchSize = (!!this.config.source.fetchSize)? parseInt(this.config.source.fetchSize) : null;
this.storeZeroValues = ( !('storeZeroValues' in this.config.target) )? true : !!this.config.target.storeZeroValues;
this.computeValuePerArea = this.config.target.computeValuePerArea;
this.sourceFeatureType;
this.targetFeatureType;
this.ohsomeApiUrl = config.ohsomeApiUrl || OHSOME_API_URL;
this.log_start = new Date();
console.log('Start at: ' + this.log_start.toLocaleString());
console.log('------------------------------');
console.log('Start Ohsome2X with:');
console.log('------------------------------');
console.log('Ohsome-API:', this.ohsomeApiUrl);
console.log(JSON.stringify(this.config.ohsomeQuery,null, 2));
console.log('------------------------------');
console.log('Source of statistical areas:');
console.log(JSON.stringify(this.config.source,null, 2));
console.log('------------------------------');
console.log('Target of statistical results:');
console.log(JSON.stringify(this.config.target,null, 2));
console.log('------------------------------');
//this.run().catch(console.log);
}
async run(){
console.log('RUN');
//initialize featureTypes with info if table aleady exists;
try {
this.sourceFeatureType = await FeatureTypeFactory.create(this.config.source);
this.targetFeatureType = await FeatureTypeFactory.create(this.config.target);
} catch (e) {
console.log(e);
throw new Error('Could not initialize FeatureTypes.');
}
//createGeometryOnTarget? default is true
const shouldCreateGeometry = (this.config.target.createGeometry == null)? true: !!this.config.target.createGeometry;
// default is false
const transformToWebmercator = ( shouldCreateGeometry && !!this.config.target.transformToWebmercator);
// write timestamp values in one or many columns? default is vertical (one column)
const shouldWriteHorizontalTimestamps = !!this.config.target.horizontalTimestampColumns;
//dropTableIfexisits and views, do not delete if completing table from cursor > 0
if (this.targetFeatureType.store.type == 'postgis' && this.cursor == 0 ){
await this.targetFeatureType.delete();
}
//createItertively or createAllAtOnce (posgis source and tagrget only)
if (! (this.sourceFeatureType instanceof PgFeatureType && this.targetFeatureType instanceof PgFeatureType) ) {
//one or both of the stores are not postgisOnly
this.fetchSize = null;
}
if (!!this.fetchSize){
//create iteratively
let cursor = this.cursor;
let featureCount = 1; // 1 to pass break test first time
try {
while (true) {
const sourceFeatureCollection = await this.sourceFeatureType.getFeaturesByCursorAndLimit(cursor, this.fetchSize);
let targetFeatureCollection;
featureCount = sourceFeatureCollection.features.length;
if(featureCount == 0){
console.log('No more cells.');
break;
}
cursor = parseInt(sourceFeatureCollection.features[sourceFeatureCollection.features.length-1].properties.id);
console.time('computeArea');
let idAreaMap;
if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
let idArea = sourceFeatureCollection.features.map((feature)=>[feature.properties.id, turfArea(feature.geometry)]);
idAreaMap = new Map(idArea);
}
console.timeEnd('computeArea');
//build bpolys and add to ohsomeQuery
//api requires ids to be strings
//sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
this.config.ohsomeQuery.bpolys = sourceFeatureCollection;
console.time('query');
let ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
console.timeEnd('query');
console.log('Start conversion ohsomeJSON to GeoJSON');
console.time('convert');
if (shouldCreateGeometry) {
if (typeof ohsomeResults.data == 'string') {
console.log('-----------------------------------------');
console.log(ohsomeResults.data.substring(0,400))
console.log('-----------------------------------------');
};
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
} else {
if (typeof ohsomeResults.data == 'string') {
console.log('-----------------------------------------');
console.log(ohsomeResults.data.substring(0,400))
console.log('-----------------------------------------');
};
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);
}
console.timeEnd('convert');
if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues){
//remove features where value = 0
console.log('Remove Zeros');
console.time('removeZero');
targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection,'value', 0);
console.timeEnd('removeZero');
}
console.time('computeValuePerArea');
if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
targetFeatureCollection.features.forEach(
(feature)=>feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
);
}
console.timeEnd('computeValuePerArea');
await this.targetFeatureType.writeFeatures(targetFeatureCollection);
//await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);
}
} catch (e) {
console.log(e);
this.sourceFeatureType.finalize();
this.targetFeatureType.finalize();
throw new Error('Could not create ohsome data.')
}
} else {
// create all at once
try {
let targetFeatureCollection;
const sourceFeatureCollection = await this.sourceFeatureType.getFeatures();
console.time('computeArea');
let idAreaMap;
if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
let idArea = sourceFeatureCollection.features.map((feature)=>[feature.properties.id, turfArea(feature.geometry)]);
idAreaMap = new Map(idArea);
}
console.timeEnd('computeArea');
//build bpolys and add to ohsomeQuery
//api requires ids to be strings
//sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
this.config.ohsomeQuery.bpolys = sourceFeatureCollection;
console.time('query');
const ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
console.timeEnd('query');
console.log('Start conversion ohsomeJSON or csv to GeoJSON');
console.time('convert');
if (shouldCreateGeometry) {
if (typeof ohsomeResults.data == 'string') {
console.log('-----------------------------------------');
console.log(ohsomeResults.data.substring(0,400))
console.log('-----------------------------------------');
};
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
} else {
if (typeof ohsomeResults.data == 'string') {
console.log('-----------------------------------------');
console.log(ohsomeResults.data.substring(0,400))
console.log('-----------------------------------------');
};
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);
}
console.timeEnd('convert');
if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues){
//remove features where value = 0
targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection,'value', 0);
}
console.time('computeValuePerArea');
if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
targetFeatureCollection.features.forEach(
(feature)=>feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
);
}
console.timeEnd('computeValuePerArea');
await this.targetFeatureType.writeFeatures(targetFeatureCollection);
//await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);
} catch (e) {
console.log(e);
this.sourceFeatureType.finalize();
this.targetFeatureType.finalize();
throw new Error('Could not create ohsome data.');
}
}
//createIndexes?
try {
if(this.targetFeatureType.store.type == 'postgis' && !!this.config.target.createIndexes){
await this.targetFeatureType.createIndex('id');
if(!!this.config.target.horizontalTimestampColumns){
// TODO:horizontal
console.log('create indexes for horizontalTimestampColumns not yet implemented!');
} else {
// vertical: one timestamp column
await this.targetFeatureType.createIndex('timestamp');
await this.targetFeatureType.clusterTable('timestamp');
await this.targetFeatureType.createIndex('value');