Commit 2d05544d authored by Michael Auer's avatar Michael Auer
Browse files

finished support for elements/.../gruopBy/boundary/groupBy/tag

parent 8cfebeff
......@@ -26,6 +26,12 @@ elements/length/ratio/groupBy/boundary
elements/area/ratio/groupBy/boundary
elements/perimeter/ratio/groupBy/boundary
//group by tags within each boundary
elements/count/groupBy/boundary/groupBy/tag
elements/length/groupBy/boundary/groupBy/tag
elements/area/groupBy/boundary/groupBy/tag
elements/perimeter/groupBy/boundary/groupBy/tag
//contributors
users/count/groupBy/boundary
```
......
......@@ -315,11 +315,19 @@ let configureOhsomeQuery = async () => {
{name: 'elements/length/groupBy/boundary'},
{name: 'elements/area/groupBy/boundary'},
{name: 'elements/perimeter/groupBy/boundary'},
{name: 'elements/count/ratio/groupBy/boundary'},
{name: 'elements/length/ratio/groupBy/boundary'},
{name: 'elements/area/ratio/groupBy/boundary'},
{name: 'elements/perimeter/ratio/groupBy/boundary'},
{name: 'elements/count/groupBy/boundary/groupBy/tag'},
{name: 'elements/length/groupBy/boundary/groupBy/tag'},
{name: 'elements/area/groupBy/boundary/groupBy/tag'},
{name: 'elements/perimeter/groupBy/boundary/groupBy/tag'},
{name: 'users/count/groupBy/boundary'}
],
default: 'elements/count/groupBy/boundary'
},
......@@ -338,6 +346,17 @@ let configureOhsomeQuery = async () => {
message: 'filter2: Specify an ohsome-api filter text. This will define the nominator value for the ratio result. (Syntax: e.g. building=* and building!=no and geometry:polygon):\n More info on syntax: https://docs.ohsome.org/ohsome-api/stable/filter.html \n>',
when: (answers) => {return answers.queryType.includes('/ratio')}
},
{
name: 'groupByKey',
message: 'groupByKey: Specify a tag-key, e.g. building, which is used to group the results \n>',
when: (answers) => {return answers.queryType.includes('/groupBy/tag')}
},
{
name: 'groupByValues',
message: 'groupByValues: Leave empty to group by all value combinations with the groupByKey which passed the filter. Or specify one or more specific tag-values, e.g. yes,no, to group specifically for e.g. building=yes and building=no. \n>',
when: (answers) => {return answers.queryType.includes('/groupBy/tag')},
filter: (input,answers)=> {return (input=='')?undefined:input;}
},
{
name: 'time',
default: '2010-01-01/2017-01-01/P1Y'
......
......@@ -8,7 +8,7 @@ import * as Papa from 'papaparse';
import {reproject} from 'reproject';
import * as proj4 from 'proj4';
import {
GeoJSONStore,
GeoJSONStore, Ohsome2XConfig, OhsomeQueryConfig,
SourceGeoJSONFeatureTypeConfig,
TargetGeoJSONFeatureTypeConfig
} from './config_types_interfaces';
......@@ -173,7 +173,7 @@ export class GeoJsonFeatureType extends FeatureType {
* @param sourceFeatureCollection if not undefined, will be used to join input geometry (bpolys) and ohsome-API results
* @param transformToWebmercator if sourceFeatureCollection is provided, should output be reprojected from WGS84 to (EPSG:4326) to Webmercator (EPSG:3857)?
**/
static fromOhsome(ohsomeGroupByBoundaryResponse: any, sourceIdType: string, horizontal: boolean, sourceFeatureCollection?: FeatureCollection, transformToWebmercator?: boolean): FeatureCollection {
static fromOhsome(ohsomeGroupByBoundaryResponse: any, ohsome2XConfig: Ohsome2XConfig, sourceIdType: string, horizontal: boolean, sourceFeatureCollection?: FeatureCollection, transformToWebmercator?: boolean): FeatureCollection {
let ohsomeResults = ohsomeGroupByBoundaryResponse;
......@@ -185,6 +185,7 @@ export class GeoJsonFeatureType extends FeatureType {
let isContributionView: boolean;
let isRatio: boolean;
let isGroupByTag: boolean = ohsome2XConfig.ohsomeQuery.queryType.includes("groupBy/tag");
if (isCSV) {
......@@ -211,6 +212,7 @@ export class GeoJsonFeatureType extends FeatureType {
// check isRatio
isRatio = ohsomeResults.groupByResult[0].result[0].ratio !== undefined;
}
// PapaParse config
......@@ -254,7 +256,22 @@ export class GeoJsonFeatureType extends FeatureType {
if (!isCSV) {
ohsomeResults.groupByResult.forEach((item: any) => {
const id = item.groupByObject;
let groupByObject = item.groupByObject;
//remove and return first element as id
const id = groupByObject.shift();
//FUTURE generic solution: rest of items could be arbitrary numbers of groupings
//current solution only for groupBy/tag
let groups:any = {};
groupByObject.forEach((group:string, index: number, array: string[])=>{
// if (array.length > 1){
// groups['group'+(index+1)] = group;
// } else {
groups['tag'] = group;
// }
});
const geom = (shouldCreateGeometry) ? idGeomMap.get(id) : null;
const getFeatureFromSnapshotResult = (tv: any) => {
return turfHelpers.feature(geom,
......@@ -287,7 +304,7 @@ export class GeoJsonFeatureType extends FeatureType {
const getFeatureFromResult = (isContributionView)? getFeatureFromContributionViewResult : (isRatio)? getFeatureFromRatioResult : getFeatureFromSnapshotResult;
const idFeatures = item.result.map(getFeatureFromResult);
const idFeatures = item.result.map(getFeatureFromResult).map((feature: Feature)=>{ Object.assign(feature.properties, groups); return feature} );
targetFeatures.push(...idFeatures);
});
......@@ -298,11 +315,13 @@ export class GeoJsonFeatureType extends FeatureType {
isContributionView = isContributionView!;
//for proper id parsing remove text-part from column names, e.g. 23_value;23_value2;23_ratio;24_value;....
if(isRatio!){
ohsomeResults = ohsomeResults.replace(/(\.*)_value(2)?/g, "$1");
}
let ohsomeCsv = Papa.parse(ohsomeResults, ohsomeGroupByBoundaryCSV).data;
const nrows = ohsomeCsv.length;
......@@ -338,6 +357,35 @@ export class GeoJsonFeatureType extends FeatureType {
properties.value = ohsomeCsv[row][valueIndex]; //value column
properties.value2 = ohsomeCsv[row][valueIndex+1]; //value2 column
properties.ratio = ohsomeCsv[row][valueIndex+2]; //ratio column
} else if (isGroupByTag){
properties = {
id: undefined,
timestamp: undefined,
tag: undefined,
value: undefined
};
let columnName:string = ohsomeCsv[0][col]; //<id>_<key>=<value> or <id>_remainder
//check if remainder
if(columnName.endsWith("_remainder")){
// @ts-ignore
properties.id = castId(columnName.match(/^(.+)_remainder$/)[1]);
properties.tag = 'remainder';
} else {
//columnName with groupByTag, like <id>_<key>=<value>
//TODO prepare it only once for all rows?
// @ts-ignore
let groupByKey = ohsome2XConfig.ohsomeQuery.groupByKey.trim();
let regExp = new RegExp(`^(.+)_(${groupByKey})=(.+)$`);
let match = columnName.match(regExp);
// @ts-ignore
properties.id = castId(match[1]);
// @ts-ignore
properties.tag = match[2]+ '=' + match[3];
}
properties.timestamp = properties.timestamp = ohsomeCsv[row][0];
properties.value = ohsomeCsv[row][col];
} else {
properties = {
id: undefined,
......@@ -369,10 +417,16 @@ export class GeoJsonFeatureType extends FeatureType {
if (!isRatio!){
targetFeatures = ohsomeResults.groupByResult.map((item: any) => {
const id = item.groupByObject;
const id = item.groupByObject.shift();
const geom = (shouldCreateGeometry) ? idGeomMap.get(id) : null;
let properties: { id: number | string; [timestamp: string]: any } = {id: id};
if(isGroupByTag){
properties.tag = item.groupByObject[0];
}
item.result.forEach((tv: any) => {
if (isContributionView){
properties[`${tv.fromTimestamp}/${tv.toTimestamp}`] = tv.value
......@@ -431,13 +485,39 @@ export class GeoJsonFeatureType extends FeatureType {
if(!isRatio!){
const id = castId(ohsomeCsv[0][col]);
const geom = (shouldCreateGeometry) ? idGeomMap!.get(id) : null;
let id;
let properties: { id: number | string | undefined; [timestamp: string]: any } = {
id: id
id: undefined
};
if(isGroupByTag){
let columnName = ohsomeCsv[0][col];
//check if remainder
if(columnName.endsWith("_remainder")){
// @ts-ignore
properties.id = castId(columnName.match(/^(.+)_remainder$/)[1]);
properties.tag = 'remainder';
} else {
//columnName with groupByTag, like <id>_<key>=<value>
//TODO prepare it only once for all rows?
// @ts-ignore
let groupByKey = ohsome2XConfig.ohsomeQuery.groupByKey.trim();
let regExp = new RegExp(`^(.+)_(${groupByKey})=(.+)$`);
let match = columnName.match(regExp);
// @ts-ignore
properties.id = castId(match[1]);
// @ts-ignore
properties.tag = match[2]+ '=' + match[3];
}
} else {
properties.id = castId(ohsomeCsv[0][col]);
}
const geom = (shouldCreateGeometry) ? idGeomMap!.get(id) : null;
for (let row = 1; row < nrows; row++) {
if (isContributionView!) {
properties[`${ohsomeCsv[row][0]}/${ohsomeCsv[row][1]}`] = ohsomeCsv[row][col]; // fromTimestamp/toTimestamp = value
......
import {
Ohsome2XConfig,
OhsomeQueryConfig,
OhsomeQueryConfigFormat, OhsomeQueryRatioConfig,
OhsomeQueryConfigFormat, OhsomeQueryGroupByTagConfig, OhsomeQueryRatioConfig,
TargetPostgisFeatureTypeConfig
} from './config_types_interfaces';
import normalizeUrl from 'normalize-url';
......@@ -33,6 +33,7 @@ class Ohsome2X extends EventEmitter {
private isContributionView: boolean | undefined;
private isGroupByResponse: boolean | undefined;
private isRatio: boolean | undefined;
private isGroupByTag: boolean | undefined;
private sourceIdType: number | string | undefined;
private totalFeatureCount: number = 0;
private currentFeatureCount: number = 0;
......@@ -56,6 +57,8 @@ class Ohsome2X extends EventEmitter {
//default use csv because its way faster than parsing json
this.config.ohsomeQuery.format = (this.config.ohsomeQuery.format == null || this.config.ohsomeQuery.format.trim() === '')? 'csv' : this.config.ohsomeQuery.format.trim() as 'csv' | 'json';
this.isGroupByTag = this.config.ohsomeQuery.queryType.includes("groupBy/tag");
this.log_start = new Date();
this.log_end = new Date();
console.log('Start at: ' + this.log_start.toLocaleString());
......@@ -254,6 +257,7 @@ class Ohsome2X extends EventEmitter {
this.isContributionView = responseFormat.isContributionView;
this.isRatio = responseFormat.isRatio;
this.harmonizeRatio_NaN_and_Infinity(ohsomeResults);
console.log('Start conversion ohsome-result to GeoJSON');
......@@ -264,7 +268,7 @@ class Ohsome2X extends EventEmitter {
console.log(ohsomeResults.data.substring(0, 500) + '...');
console.log('-----------------------------------------');
}
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.config, this.sourceIdType, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
} else {
if (typeof ohsomeResults.data == 'string') {
......@@ -272,7 +276,7 @@ class Ohsome2X extends EventEmitter {
console.log(ohsomeResults.data.substring(0, 500) + '...');
console.log('-----------------------------------------');
}
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps);
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.config, this.sourceIdType, shouldWriteHorizontalTimestamps);
}
console.timeEnd('convert');
......@@ -370,7 +374,7 @@ class Ohsome2X extends EventEmitter {
console.log(ohsomeResults.data.substring(0, 400) + '...');
console.log('-----------------------------------------');
}
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.config, this.sourceIdType, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
} else {
......@@ -379,7 +383,7 @@ class Ohsome2X extends EventEmitter {
console.log(ohsomeResults.data.substring(0, 400) + '...');
console.log('-----------------------------------------');
}
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps);
targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.config, this.sourceIdType, shouldWriteHorizontalTimestamps);
}
console.timeEnd('convert');
......@@ -431,7 +435,7 @@ class Ohsome2X extends EventEmitter {
// TODO:horizontal
console.log('create indexes for horizontalTimestampColumns not yet implemented!');
} else {
// vertical: one timestamp column
// vertical: one/two timestamp column(s)
if (this.isContributionView){
await this.targetFeatureType.createIndex('from_timestamp');
......@@ -455,6 +459,10 @@ class Ohsome2X extends EventEmitter {
await this.targetFeatureType.createIndex('ratio');
}
if(this.isGroupByTag){
await this.targetFeatureType.createIndex('tag');
}
if (this.computeValuePerArea && !this.isRatio) {
await this.targetFeatureType.createIndex('value_per_area');
}
......@@ -495,13 +503,19 @@ class Ohsome2X extends EventEmitter {
///////////// methods
async getOhsomeResults(ohsomeQuery: OhsomeQueryConfig | OhsomeQueryRatioConfig): Promise<AxiosResponse> {
async getOhsomeResults(ohsomeQuery: OhsomeQueryConfig | OhsomeQueryRatioConfig | OhsomeQueryGroupByTagConfig): Promise<AxiosResponse> {
const isRatio = ohsomeQuery.queryType.includes("/ratio");
const isGroupByTag = ohsomeQuery.queryType.includes("/groupBy/tag");
// set empty strings to null
const filter = (ohsomeQuery.filter != null && ohsomeQuery.filter.trim() != '')? ohsomeQuery.filter.trim() : undefined;
// for ratio requests
const filter2 = (isRatio && ohsomeQuery.filter2 != null && ohsomeQuery.filter2.trim() != '')? ohsomeQuery.filter2.trim() : undefined;
//for groupByTags requests
const groupByKey = (isGroupByTag && ohsomeQuery.groupByKey != null && ohsomeQuery.groupByKey.trim() != '')? ohsomeQuery.groupByKey.trim() : undefined;
const groupByValues = (isGroupByTag && ohsomeQuery.groupByValues != null && ohsomeQuery.groupByValues.trim() != '')? ohsomeQuery.groupByValues.trim() : undefined;
const format: OhsomeQueryConfigFormat = (ohsomeQuery.format != null && ohsomeQuery.format.trim() != '')? ohsomeQuery.format.trim() as 'csv'|'json' : undefined;
try {
......@@ -516,6 +530,8 @@ class Ohsome2X extends EventEmitter {
bpolys: bpolyString,
filter: filter,
filter2: filter2,
groupByKey: groupByKey,
groupByValues: groupByValues,
time: ohsomeQuery.time,
showMetadata: true,
format: format
......@@ -527,13 +543,14 @@ class Ohsome2X extends EventEmitter {
// always make a group by boundary query for backward compatibility
// complete ohsomeAPI resource path should be used
let queryType = ohsomeQuery.queryType.replace(/(.*)(\/)?(\/groupBy\/boundary)(\/)?/, "$1");
// let queryType = ohsomeQuery.queryType.replace(/(.*)(\/)?(\/groupBy\/boundary)(\/)?/, "$1");
const queryType = ohsomeQuery.queryType;
console.log(`Querying ohsome-API: ${this.ohsomeApiUrl}/${queryType}/groupBy/boundary`);
console.log(`Querying ohsome-API: ${this.ohsomeApiUrl}/${queryType}`);
console.log('with params:', Object.entries(cleanDataObject).reduce((a,[k,v])=>({...a, [k]: (typeof v =='string')?v.substring(0,200): v }),{}));
const stats = await axios(<AxiosRequestConfig>{
url: `${this.ohsomeApiUrl}/${queryType}/groupBy/boundary`,
url: `${this.ohsomeApiUrl}/${queryType}`,
method: 'post',
header: {'content-type': 'application/x-www-form-urlencoded'},
maxContentLength: 1024 * 1024 * 1024 * 1024,
......@@ -617,6 +634,15 @@ class Ohsome2X extends EventEmitter {
})
}
}
// standard groupBy into array-based multigroupBy as in groupBy/boundary/groupBy/tag
if(ohsomeResults.data.groupByResult && !Array.isArray(ohsomeResults.data.groupByResult.groupByObject) ){
ohsomeResults.data.groupByResult.forEach((groupByResult: any)=>{
if(!Array.isArray(groupByResult.groupByObject)){
groupByResult.groupByObject = [groupByResult.groupByObject];
}
})
}
}
// end of response harmonization
......
......@@ -212,6 +212,7 @@ export class PgFeatureType extends FeatureType {
case 'value_per_area':
return 'double precision';
case 'attrName':
case 'tag':
return 'text';
//timestamp or timerange in horizontal mode as columnNames
case (columnName.split('/').every((datetime)=>moment(datetime).isValid()))? columnName: null:
......@@ -283,6 +284,7 @@ export class PgFeatureType extends FeatureType {
case 'fromTimestamp':
case 'toTimestamp':
case 'attrName':
case 'tag':
return `'${value}'`;
case 'node':
case 'way':
......
......@@ -62,8 +62,10 @@ export type OhsomeQueryConfigFormat = 'json' | 'csv' | undefined;
export interface OhsomeQueryConfig {
queryType: string; // e.g. elements/count for elements/count/groupBy/boundary
filter: string // new ohsome-api filter format see: https://docs.ohsome.org/ohsome-api/
filter2?: string // only for ratio requests
filter: string; // new ohsome-api filter format see: https://docs.ohsome.org/ohsome-api/
filter2?: string; // only for ratio requests
groupByKey?: string; // only for groupBy/tag requests
groupByValues?: string; // only for groupBy/tag requests
time: string; // e.g. ISO8601 start/end/interval_period '2015-01-01/2017-01-01/P1Y'
bpolys?: FeatureCollection | string;
bboxes?: string;
......@@ -76,6 +78,11 @@ export interface OhsomeQueryRatioConfig extends OhsomeQueryConfig{
filter2: string //defines the value2 in the result data (value2/value=ratio), uses same format as filter see: https://docs.ohsome.org/ohsome-api/
}
export interface OhsomeQueryGroupByTagConfig extends OhsomeQueryConfig{
groupByKey: string;
groupByValues?: string
}
export interface Ohsome2XConfig {
ohsomeApiUrl?: string;
source: SourceGeoJSONFeatureTypeConfig | SourcePostgisFeatureTypeConfig;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment