Skip to content

GitLab

Commit f05adece authored by Michael Auer's avatar Michael Auer
Browse files

library support for contribution view queries 

#3
parent 48a1def6
Expand all Hide whitespace changes
Inline Side-by-side
package-lock.json
View file @ f05adece
......@@ -222,6 +222,15 @@
"through": "^2.3.8"
}
},
"exec-sh": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.2.2.tgz",
"integrity": "sha512-FIUCJz1RbuS0FKTdaAafAByGS0CPvU3R0MeHxgtl+djzCc//F8HakL8GzmVNZanasTbTAY/3DRFA0KpVqj/eAw==",
"dev": true,
"requires": {
"merge": "^1.2.0"
}
},
"external-editor": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz",
......@@ -340,6 +349,12 @@
"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz",
"integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg="
},
"merge": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz",
"integrity": "sha512-VjFo4P5Whtj4vsLzsYBu5ayHhoHJ0UqNm7ibvShmbmoz7tGi0vXaoJbGdB+GmDMLUdg8DpQXEIeVDAe8MaABvQ==",
"dev": true
},
"mgrs": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/mgrs/-/mgrs-1.0.0.tgz",
......@@ -659,6 +674,16 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"watch": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/watch/-/watch-1.0.2.tgz",
"integrity": "sha1-NApxe952Vyb6CqB9ch4BR6VR3ww=",
"dev": true,
"requires": {
"exec-sh": "^0.2.0",
"minimist": "^1.2.0"
}
},
"wkt-parser": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/wkt-parser/-/wkt-parser-1.2.3.tgz",
......
package.json
View file @ f05adece
......@@ -4,6 +4,7 @@
"description": "convenience library to query ohsome-api using nodejs",
"main": "Ohsome2X.js",
"scripts": {
"watch": "watch 'npm run build' src",
"test": "echo \"Error: no test specified\" && exit 1",
"build": "tsc --build ./tsconfig.json"
},
......@@ -36,6 +37,7 @@
"@types/papaparse": "^5.0.0",
"@types/pg": "^7.4.14",
"@types/proj4": "^2.5.0",
"typescript": "^3.5.3"
"typescript": "^3.5.3",
"watch": "^1.0.2"
}
}
src/GeoJsonFeatureType.ts
View file @ f05adece
......@@ -9,6 +9,7 @@ import {reproject} from 'reproject';
import * as proj4 from 'proj4';
import {GeoJSONFeatureTypeConfig, GeoJSONStore} from './config_types_interfaces';
//const fs = require('fs');
//var path = require('path');
//const turfHelpers = require('@turf/helpers');
......@@ -25,96 +26,96 @@ export class GeoJsonFeatureType extends FeatureType {
private path: string;
private fileExists: boolean;
constructor(config: GeoJSONFeatureTypeConfig){
super(config);
this.store = config.store; //type:'geojson'
this.name = config.name || path.basename(config.store.path);
this.geometryId = config.geometryId || 'id';
this.path = config.store.path;
this.fileExists = fs.existsSync(this.path);
console.log('file exists: ' + this.fileExists);
}
async delete(){
if(this.fileExists){
try {
console.log('DELETE GeoJSON file: ' + this.path);
fs.unlinkSync(this.path);
//file removed
} catch(err) {
console.error(err)
}
}
};
/**
* @returns GeoJSON<FeatureCollection>
**/
getFeatures(){
if(!fs.existsSync(this.path)){
throw new Error('Cannot get features. File does not exist: ' + this.path);
}
let geojson;
try {
let rawdata = <string>fs.readFileSync(this.path,'utf-8');
geojson = JSON.parse(rawdata);
} catch (e) {
console.log(e);
constructor(config: GeoJSONFeatureTypeConfig) {
super(config);
this.store = config.store; //type:'geojson'
this.name = config.name || path.basename(config.store.path);
this.geometryId = config.geometryId || 'id';
this.path = config.store.path;
this.fileExists = fs.existsSync(this.path);
console.log('file exists: ' + this.fileExists);
}
try {
turfInvariant.geojsonType(geojson, 'FeatureCollection', 'getFeatures');
} catch (e) {
console.log(e);
throw new Error('GeoJSON must be of type FeatureCollection.');
}
async delete() {
if (this.fileExists) {
try {
console.log('DELETE GeoJSON file: ' + this.path);
fs.unlinkSync(this.path);
//file removed
} catch (err) {
console.error(err)
}
}
};
//normalize id columnName
if(this.geometryId != 'id'){
geojson.features.forEach(
(feature: Feature)=>{
if(feature.properties === null || !feature.properties[this.geometryId]){
throw new Error(`No id column "${this.geometryId}" found on ${this.name}. Please configure geometryId: '<columnNameOfId>' in featureTypeConfig.`);
/**
* @returns GeoJSON<FeatureCollection>
**/
getFeatures() {
if (!fs.existsSync(this.path)) {
throw new Error('Cannot get features. File does not exist: ' + this.path);
}
let geojson;
try {
let rawdata = <string>fs.readFileSync(this.path, 'utf-8');
geojson = JSON.parse(rawdata);
} catch (e) {
console.log(e);
}
feature.properties.id = feature.properties[this.geometryId];
delete feature.properties[this.geometryId]
},this);
}
return geojson;
try {
turfInvariant.geojsonType(geojson, 'FeatureCollection', 'getFeatures');
} catch (e) {
console.log(e);
throw new Error('GeoJSON must be of type FeatureCollection.');
}
//normalize id columnName
if (this.geometryId != 'id') {
geojson.features.forEach(
(feature: Feature) => {
if (feature.properties === null || !feature.properties[this.geometryId]) {
throw new Error(`No id column "${this.geometryId}" found on ${this.name}. Please configure geometryId: '<columnNameOfId>' in featureTypeConfig.`);
}
feature.properties.id = feature.properties[this.geometryId];
delete feature.properties[this.geometryId]
}, this);
}
return geojson;
}
async writeFeatures(featureCollection: FeatureCollection) {
//do nothing when feature collection is empty
if (featureCollection.features.length == 0) {
console.log('WARNING: No features in feature collection. No feature writing.');
return;
}
let data = JSON.stringify(featureCollection);
try {
console.log(`Saving GeoJSON file to: ${this.path}`);
fs.writeFileSync(this.path, data);
} catch (e) {
console.log(e);
async writeFeatures(featureCollection: FeatureCollection) {
//do nothing when feature collection is empty
if (featureCollection.features.length == 0) {
console.log('WARNING: No features in feature collection. No feature writing.');
return;
}
let data = JSON.stringify(featureCollection);
try {
console.log(`Saving GeoJSON file to: ${this.path}`);
fs.writeFileSync(this.path, data);
} catch (e) {
console.log(e);
}
}
}
static removeFeaturesByPropertyValue(featureCollection: FeatureCollection,propertyName: string, value:any){
static removeFeaturesByPropertyValue(featureCollection: FeatureCollection, propertyName: string, value: any) {
let filteredFeatures = featureCollection.features.filter(
(feature) => {
if (feature.properties === null) {
return true; //keep features lacking the indicated property
} else {
return feature.properties[propertyName] !== value;
}
}
);
featureCollection.features = filteredFeatures;
return featureCollection;
}
let filteredFeatures = featureCollection.features.filter(
(feature) => {
if (feature.properties === null) {
return true; //keep features lacking the indicated property
} else {
return feature.properties[propertyName] !== value;
}
}
);
featureCollection.features = filteredFeatures;
return featureCollection;
}
/**
......@@ -124,148 +125,215 @@ export class GeoJsonFeatureType extends FeatureType {
* @param sourceFeatureCollection if not undefined, will be used to join input geometry (bpolys) and ohsome-API results
* @param transformToWebmercator if sourceFeatureCollection is provided, should output be reprojected from WGS84 to (EPSG:4326) to Webmercator (EPSG:3857)?
**/
static fromOhsome(ohsomeGroupByBoundaryResponse: any, horizontal: boolean, sourceFeatureCollection?: FeatureCollection, transformToWebmercator?: boolean): FeatureCollection {
const ohsomeGroupByBoundaryCSV = {
delimiter: ';',
dynamicTyping: (col:any)=>(col != 0), //first column is timestamp and should remain string not auto converted to Date
header: false,
skipEmptyLines: true,
comments: '#',
newline: '\n'
};
static fromOhsome(ohsomeGroupByBoundaryResponse: any, horizontal: boolean, sourceFeatureCollection?: FeatureCollection, transformToWebmercator?: boolean): FeatureCollection {
const ohsomeResults = ohsomeGroupByBoundaryResponse;
const ohsomeResults = ohsomeGroupByBoundaryResponse;
const isCSV = (typeof ohsomeResults == 'string');
const isCSV = (typeof ohsomeResults == 'string');
let isContributionView: boolean = true;
const shouldCreateGeometry = !!sourceFeatureCollection;
let targetFeatures: Feature[] = [];
let targetFeatureCollection: FeatureCollection;
if (isCSV) {
const lines = ohsomeResults.split("\n");
//find first non-comment line
for (let i = 0; i < lines.length; i++) {
if (!lines[i].trimLeft().startsWith("#")) {
// check columnheaders for contributionview column names
const columns = lines[i].trim().split(";");
isContributionView = columns.includes("fromTimestamp") && columns.includes("toTimestamp");
break;
}
}
} else {
//JSON Response
const hasFromTimestamps = ohsomeResults.groupByResult[0].result[0].fromTimestamp !== undefined;
const hasToTimestamps = ohsomeResults.groupByResult[0].result[0].toTimestamp !== undefined;
isContributionView = hasFromTimestamps && hasToTimestamps;
}
let idGeomMap: Map<any, any>;
if(shouldCreateGeometry){
//create idGeomMap
idGeomMap = new Map();
const ohsomeGroupByBoundaryCSV = {
delimiter: ';',
dynamicTyping: (isContributionView) ? (col: any) => (!(col == 0 || col == 1)) : (col: any) => (col != 0), //first (or first two) column is timestamp and should remain string not auto converted to Date
header: false,
skipEmptyLines: true,
comments: '#',
newline: '\n'
};
if(transformToWebmercator){
console.log('TRANSFORMING...');
sourceFeatureCollection = <FeatureCollection>reproject(<FeatureCollection>sourceFeatureCollection, proj4.WGS84, "EPSG:3857", proj4.defs);
}
const shouldCreateGeometry = !!sourceFeatureCollection;
let targetFeatures: Feature[] = [];
let targetFeatureCollection: FeatureCollection;
// @ts-ignore
sourceFeatureCollection.features.forEach((feature) => idGeomMap.set(feature.properties.id, feature.geometry));
}
let idGeomMap: Map<any, any>;
if (shouldCreateGeometry) {
//create idGeomMap
idGeomMap = new Map();
if (!horizontal){
//vertical is the default if undefined
if (transformToWebmercator) {
console.log('TRANSFORMING...');
// ohsomeJSON
if(!isCSV){ohsomeResults.groupByResult.forEach((item: any)=>{
sourceFeatureCollection = <FeatureCollection>reproject(<FeatureCollection>sourceFeatureCollection, proj4.WGS84, "EPSG:3857", proj4.defs);
}
const id = item.groupByObject;
const geom = (shouldCreateGeometry)? idGeomMap.get(id) : null;
const idFeatures = item.result.map((tv: any)=>{
return turfHelpers.feature(geom,
{
id: id,
timestamp: tv.timestamp,
value: tv.value
})
});
// @ts-ignore
sourceFeatureCollection.features.forEach((feature) => idGeomMap.set(feature.properties.id, feature.geometry));
}
targetFeatures.push(...idFeatures);
});}
if (!horizontal) {
//vertical is the default if undefined
// ohsomeJSON
if (!isCSV) {
ohsomeResults.groupByResult.forEach((item: any) => {
const id = item.groupByObject;
const geom = (shouldCreateGeometry) ? idGeomMap.get(id) : null;
const getFeatureFromSnapshotResult = (tv: any) => {
return turfHelpers.feature(geom,
{
id: id,
timestamp: tv.timestamp,
value: tv.value
})
};
const getFeatureFromContributionViewResult = (tv: any) => {
return turfHelpers.feature(geom,
{
id: id,
from_timestamp: tv.fromTimestamp,
to_timestamp: tv.toTimestamp,
value: tv.value
})
}
if(isCSV){
//csv
const getFeatureFromResult = (isContributionView)? getFeatureFromContributionViewResult : getFeatureFromSnapshotResult;
let ohsomeCsv = Papa.parse(ohsomeResults,ohsomeGroupByBoundaryCSV).data;
const idFeatures = item.result.map(getFeatureFromResult);
let nrows = ohsomeCsv.length;
let ncols = ohsomeCsv[0].length;
targetFeatures.push(...idFeatures);
});
}
for (let col = 1; col < ncols; col++) {
for (let row = 1; row < nrows; row++) {
if (isCSV) {
//csv
let properties = {
timestamp: undefined,
id: undefined,
value: undefined
};
properties.timestamp = ohsomeCsv[row][0];
properties.id = ohsomeCsv[0][col];
properties.value = ohsomeCsv[row][col];
let ohsomeCsv = Papa.parse(ohsomeResults, ohsomeGroupByBoundaryCSV).data;
// @ts-ignore
const geom = (shouldCreateGeometry)? idGeomMap.get(properties.id) : null;
const nrows = ohsomeCsv.length;
const ncols = (!!ohsomeCsv[0]) ? ohsomeCsv[0].length : 0;
const firstValueColumnIndex = (isContributionView) ? 2 : 1; // for snapshotView start reading values at column index 1, for contributionView start at column index 2
targetFeatures.push(turfHelpers.feature(geom,properties));
for (let col = firstValueColumnIndex; col < ncols; col++) {
for (let row = 1; row < nrows; row++) {
let properties;
if (isContributionView) {
properties = {
id: undefined,
from_timestamp: undefined,
to_timestamp: undefined,
value: undefined
};
properties.from_timestamp = ohsomeCsv[row][0];
properties.to_timestamp = ohsomeCsv[row][1];
properties.id = ohsomeCsv[0][col];
properties.value = ohsomeCsv[row][col];
} else {
properties = {
id: undefined,
timestamp: undefined,
value: undefined
};
properties.timestamp = ohsomeCsv[row][0];
properties.id = ohsomeCsv[0][col];
properties.value = ohsomeCsv[row][col];
}
// @ts-ignore
const geom = (shouldCreateGeometry) ? idGeomMap.get(properties.id) : null;
targetFeatures.push(turfHelpers.feature(geom, properties));
}
}
}
}
} else {
//horizontal
} else {
//horizontal
if (!isCSV) {
if(!isCSV){
targetFeatures = ohsomeResults.groupByResult.map((item: any) => {
targetFeatures = ohsomeResults.groupByResult.map((item: any)=>{
const id = item.groupByObject;
const geom = (shouldCreateGeometry) ? idGeomMap.get(id) : null;
const id = item.groupByObject;
const geom = (shouldCreateGeometry)? idGeomMap.get(id) : null;
let properties: { id: number | string; [timestamp: string]: any } = {id: id};
item.result.forEach((tv: any) => {
if (isContributionView){
properties[`${tv.fromTimestamp}/${tv.toTimestamp}`] = tv.value
} else {
properties[tv.timestamp] = tv.value
}
let properties:{id:number|string;[timestamp:string]:any} = {id: id};
item.result.forEach((tv:any)=>{properties[tv.timestamp] = tv.value});
return turfHelpers.feature( geom, properties);
});
}
});
return turfHelpers.feature(geom, properties);
});
}
if(isCSV){
//csv
if (isCSV) {
//csv
let ohsomeCsv = Papa.parse(ohsomeResults,ohsomeGroupByBoundaryCSV).data;
const ohsomeCsv = Papa.parse(ohsomeResults, ohsomeGroupByBoundaryCSV).data;
let nrows = ohsomeCsv.length;
let ncols = (!!ohsomeCsv[0])? ohsomeCsv[0].length : 0;
const nrows = ohsomeCsv.length;
const ncols = (!!ohsomeCsv[0]) ? ohsomeCsv[0].length : 0;
const firstValueColumnIndex = (isContributionView) ? 2 : 1; // for snapshotView start reading values at column indeex 1, for contributionView start at column index 2
for (let col = 1; col < ncols; col++) {
for (let col = firstValueColumnIndex; col < ncols; col++) {
let properties:{id:number|string|undefined;[timestamp:string]:any} = {
let properties: { id: number | string | undefined; [timestamp: string]: any } = {
id: undefined
};
properties.id = ohsomeCsv[0][col];
for (let row = 1; row < nrows; row++) {
properties[ohsomeCsv[row][0]] = ohsomeCsv[row][col]; // timestamp = value
if (isContributionView) {
properties[`${ohsomeCsv[row][0]}/${ohsomeCsv[row][1]}`] = ohsomeCsv[row][col]; // fromTimestamp/toTimestamp = value
} else {
properties[ohsomeCsv[row][0]] = ohsomeCsv[row][col]; // timestamp = value
}
}
// @ts-ignore
const geom = (shouldCreateGeometry)? idGeomMap.get(properties.id) : null;
const geom = (shouldCreateGeometry) ? idGeomMap.get(properties.id) : null;
targetFeatures.push(turfHelpers.feature(geom,properties));
targetFeatures.push(turfHelpers.feature(geom, properties));
}
}
}
}
}
targetFeatureCollection = turfHelpers.featureCollection(targetFeatures);
targetFeatureCollection = turfHelpers.featureCollection(targetFeatures);
if (shouldCreateGeometry && transformToWebmercator){
// @ts-ignore
targetFeatureCollection.crs = {"type":"name","properties":{"name":"EPSG:3857"}};
}
if (shouldCreateGeometry && transformToWebmercator) {
// @ts-ignore
targetFeatureCollection.crs = {"type": "name", "properties": {"name": "EPSG:3857"}};
}
return targetFeatureCollection;
}
return targetFeatureCollection;
}
writeOhsomeFeatures(ohsomeGroupByBoundaryResponseJSON: any, horizontalTimestampColumns: boolean): Promise<void> {
throw new Error("Method not implemented.");
}
finalize(): Promise<void> {
throw new Error("Method not implemented.");
return Promise.resolve();
}
}
src/Ohsome2X.ts
View file @ f05adece
This diff is collapsed.
src/PgFeatureType.ts
View file @ f05adece
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment