Ohsome2X.ts 32.8 KB
Newer Older
1
2
import {
    Ohsome2XConfig,
3
    OhsomeQueryConfig,
4
    OhsomeQueryConfigFormat, OhsomeQueryRatioConfig,
5
6
    TargetPostgisFeatureTypeConfig
} from './config_types_interfaces';
7
8
9
10
11
12
13
import normalizeUrl from 'normalize-url';
import {GeoJsonFeatureType} from './GeoJsonFeatureType';
import {PgFeatureType} from './PgFeatureType';
import {FeatureTypeFactory} from './FeatureTypeFactory';
import {Feature, Geometry} from '@turf/helpers';
import turfArea from '@turf/area';
import * as querystring from 'querystring';
14
import axios, {AxiosError, AxiosRequestConfig, AxiosResponse} from 'axios';
15

16
import defaultConfig from './conf/default';
17
import {EventEmitter} from 'events';
18
import Ohsome2XError from './Ohsome2XError';
19

20
21
let OHSOME_API_URL = normalizeUrl(defaultConfig.OHSOME_API_URL); //remove trailing slash and other things

22
class Ohsome2X extends EventEmitter {
23
    private config: Ohsome2XConfig;
24
    private cursor: number | string | null | undefined;
25
26
27
28
29
30
31
32
33
    private fetchSize: number | null;
    private storeZeroValues: boolean;
    private computeValuePerArea: boolean | undefined;
    private sourceFeatureType: GeoJsonFeatureType | PgFeatureType | null;
    private targetFeatureType: GeoJsonFeatureType | PgFeatureType | null;
    private ohsomeApiUrl: string;
    private log_start: Date;
    private log_end: Date;
    private isContributionView: boolean | undefined;
34
35
36
    private isGroupByResponse: boolean | undefined;
    private isRatio: boolean | undefined;
    private sourceIdType: number | string | undefined;
37
38
    private totalFeatureCount: number = 0;
    private currentFeatureCount: number = 0;
39
40

    constructor(config: Ohsome2XConfig) {
41
        super(); //EventEmitter
42

43
        this.config = Object.assign({}, config);
44
45
46

        this.cursor = ('cursor' in this.config.source)? this.config.source.cursor : null;

47
48
49
50
51
52
53
54
        // @ts-ignore fetchSize only available for PostgisStoreConfig
        this.fetchSize = (!!this.config.source.fetchSize) ? parseInt(this.config.source.fetchSize) : null;
        this.storeZeroValues = (!('storeZeroValues' in this.config.target)) ? true : !!this.config.target.storeZeroValues;
        this.computeValuePerArea = this.config.target.computeValuePerArea;
        this.sourceFeatureType = null;
        this.targetFeatureType = null;

        this.ohsomeApiUrl = config.ohsomeApiUrl || OHSOME_API_URL;
55
56
57
58

        //default use csv because its way faster than parsing json
        this.config.ohsomeQuery.format = (this.config.ohsomeQuery.format == null || this.config.ohsomeQuery.format.trim() === '')? 'csv' : this.config.ohsomeQuery.format.trim() as 'csv' | 'json';

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
        this.log_start = new Date();
        this.log_end = new Date();
        console.log('Start at: ' + this.log_start.toLocaleString());
        console.log('------------------------------');
        console.log('Start Ohsome2X with:');
        console.log('------------------------------');
        console.log('Ohsome-API:', this.ohsomeApiUrl);
        console.log(JSON.stringify(this.config.ohsomeQuery, null, 2));
        console.log('------------------------------');
        console.log('Source of statistical areas:');
        console.log(JSON.stringify(this.config.source, null, 2));
        console.log('------------------------------');
        console.log('Target of statistical results:');
        console.log(JSON.stringify(this.config.target, null, 2));
        console.log('------------------------------');
        //this.run().catch(console.log);
75
76
    }

77
78
    async run() {
        console.log('RUN');
79

80
81
82
83
        //initialize featureTypes with info if table aleady exists;
        try {
            this.sourceFeatureType = await FeatureTypeFactory.create(this.config.source);
            this.targetFeatureType = await FeatureTypeFactory.create(this.config.target);
84
85

            this.totalFeatureCount = await this.sourceFeatureType.getFeatureCount();
86
            this.sourceIdType = await this.sourceFeatureType.getIdJsType();
87
88
89
90
91
92
93
94

            process.on('SIGINT', async ()=>{
                console.log('\nSIGINT signal received. Closing existing connections.');
                this.sourceFeatureType!.finalize();
                await this.targetFeatureType!.finalize();
                process.exit(0);
            })

95
        } catch (e) {
96
97
98
99
            //console.log(e);
           // this.emit('error', {type:'error', error: e, message: 'Could not initialize FeatureTypes.', config: this.config, cursor: this.cursor, timestamp: new Date().toISOString()});
            throw new Ohsome2XError('Could not initialize FeatureTypes.', {cursor: this.cursor},  e);
          //  throw new Error('Could not initialize FeatureTypes.');
100
        }
101

102
        //createGeometryOnTarget? default is true
103
        const shouldCreateGeometry = (this.config.target.createGeometry == null) ? true : this.config.target.createGeometry;
104
105
        // default is false
        const transformToWebmercator = (shouldCreateGeometry && !!this.config.target.transformToWebmercator);
106

107
108
        // write timestamp values in one or many columns? default is vertical (one column or two columns for contribution view queries)
        const shouldWriteHorizontalTimestamps = !!this.config.target.horizontalTimestampColumns;
109

110
        //dropTableIfexisits and views, do not delete if completing table from explicitly specified cursor
111
        // @ts-ignore
112
        if (this.targetFeatureType.store.type == 'postgis' && this.cursor == null) {
113
114
115
            try {
                await this.targetFeatureType.delete();
            } catch (e) {
116
117
118
119
                // console.log(e);
                // this.emit('error', {type:'error', error: e, message: 'Could not delete existing PostGIS table.', config: this.config, cursor: this.cursor, timestamp: new Date().toISOString()});
                throw new Ohsome2XError('Could not delete existing PostGIS table.', {cursor: this.cursor},  e);
                // throw new Error('Could not delete existing PostGIS table.');
120
121
            }

122
123
124
125
126
127
        }

        //createItertively or createAllAtOnce (posgis source and tagrget only)
        if (!(this.sourceFeatureType instanceof PgFeatureType && this.targetFeatureType instanceof PgFeatureType)) {
            //one or both of the stores are not postgisOnly => can't use fetch size, no chunkwise processing possible
            this.fetchSize = null;
128
        }
129
130
131
132
133
134
135

        if (!!this.fetchSize) {
            // chunkwise processing
            this.sourceFeatureType = <PgFeatureType>this.sourceFeatureType;
            this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
            //create iteratively
            //check curser type: number or string?
136
            const cursorType = this.sourceIdType;
137
138
139
140
141
142
143
144
145
146
147
148
149
            console.log("cursorType", cursorType);

            let cursor;
            //cursor was defined in config
            if (this.cursor != null){
                cursor = this.cursor;
            } else if (cursorType === 'number'){
                cursor = Number.MIN_SAFE_INTEGER;
            } else {
                //cursorType == 'string'
                cursor = '';
            }

150
151
152
153
154
            let fetchSize; //might be adjusted automatically if query takes too long
            fetchSize = this.fetchSize;

            let timesFetchSizeReduced = 0;

155
156
157
158
            let featureCount = 1; // 1 to pass break test first time
            try {
                while (true) {

159
                    this.emit('progress', {type:'progress', processed: this.currentFeatureCount, total: this.totalFeatureCount, cursor: cursor, fetchSize: fetchSize, timestamp: new Date().toISOString()});
160

161
                    const sourceFeatureCollection: any = await this.sourceFeatureType.getFeaturesByCursorAndLimit(cursor, fetchSize);
162
                    let targetFeatureCollection: any;
163
164

                    featureCount = sourceFeatureCollection.features.length;
165

166
167
168
169
                    if (featureCount == 0) {
                        console.log('No more cells.');
                        break;
                    }
170

171
172
173
174
175
176
177
178
179
180
181
182
183
184

                    console.time('computeArea');
                    let idAreaMap: Map<any, number>;
                    if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea) {
                        // @ts-ignore
                        let idArea: [any, number][] = sourceFeatureCollection.features.map((feature: Feature) => [feature.properties.id, turfArea(<Geometry>feature.geometry)]);
                        idAreaMap = new Map(idArea);
                    }
                    console.timeEnd('computeArea');

                    //build bpolys and add to ohsomeQuery
                    this.config.ohsomeQuery.bpolys = sourceFeatureCollection;


185
186
187
188
                    let ohsomeResults: AxiosResponse<any>;
                    try {

                        let queryStart = Date.now();
Michael Auer's avatar
Michael Auer committed
189
190
                        ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
                        // ohsomeResults = await Promise.reject({isAxiosError: true, response: {status: 413, statusText:'Payload too large', data:{}, headers:{}}, code: 'ERRTIMEOUT', config:{data:{}}});
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
                        let queryEnd = Date.now();
                        let queryDuration = queryEnd - queryStart; //duration in milliseconds
                        console.log('Query duration (sec)', (queryDuration/1000));

                        //recover reduced fetch size if response times are lower than 4 minutes
                        if (fetchSize < this.fetchSize && queryDuration <= 4 * 60 * 1000){
                            timesFetchSizeReduced--;
                            timesFetchSizeReduced = Math.max(0,timesFetchSizeReduced); //should never get negative, 0 means original fetchSize
                            fetchSize = Math.min(this.fetchSize, Math.round(this.fetchSize / (2 ** timesFetchSizeReduced)));
                            console.log('INCREASING fetchSize...');
                            console.log('timesFetchSizeReduced', timesFetchSizeReduced);
                            console.log('currentFetchSize', fetchSize, 'originalFetchSize', this.fetchSize);
                        }


                    } catch (e) {
                        if(this.config.ohsomeQuery.bpolys) {
                            delete this.config.ohsomeQuery.bpolys; //avoid logging
                        }
210

211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
                        if (e.isAxiosError) {
                            let axiosError: AxiosError = e;
                            delete axiosError.config.data; //avoid logging bpoly data
                            if (axiosError.response) {
                                if (axiosError.response.status == 413) {
                                    //try reducing fetch size as long as possible before failing
                                    //check if minimum fetch size alredy reached
                                    if (fetchSize <= 1) {
                                        throw new Ohsome2XError('Could not get Ohsome results, even with fetchSize: 1. Processing took too long with current query and service configuration.', {}, axiosError);
                                    }

                                    //payload too large: reducing fetch size
                                    timesFetchSizeReduced++;
                                    fetchSize = Math.max(1, Math.round(this.fetchSize / (2 ** timesFetchSizeReduced)));
                                    console.log('REDUCING fetchSize...');
                                    console.log('timesFetchSizeReduced', timesFetchSizeReduced);
                                    console.log('currentFetchSize', fetchSize, 'originalFetchSize', this.fetchSize);
                                    continue; //restart loop with reduced fetchSize
                                } else {
                                    //other error codes
                                    throw axiosError;
                                }
                            }  else if (axiosError.request) {
                                // The request was made but no response was received
                                // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
                                // http.ClientRequest in node.js
                                console.log('No response received.');
                                throw axiosError;
                            }

                        } else {
                            //no AxiosError something esle has happened
                            throw e;
                        }

                    }

                    ohsomeResults = ohsomeResults!;
249
250
251
252
253
254
255
256
257

                    //temporary until ohsome-api changes
                    this.harmonizeResponseStructure(ohsomeResults);

                    let responseFormat = this.checkResponseFormat (ohsomeResults.data);
                    this.isContributionView = responseFormat.isContributionView;
                    this.isRatio = responseFormat.isRatio;

                    this.harmonizeRatio_NaN_and_Infinity(ohsomeResults);
258

259
                    console.log('Start conversion ohsome-result to GeoJSON');
260
261
262
263
                    console.time('convert');
                    if (shouldCreateGeometry) {
                        if (typeof ohsomeResults.data == 'string') {
                            console.log('-----------------------------------------');
264
                            console.log(ohsomeResults.data.substring(0, 500) + '...');
265
266
                            console.log('-----------------------------------------');
                        }
267
                        targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
268
269
270
271

                    } else {
                        if (typeof ohsomeResults.data == 'string') {
                            console.log('-----------------------------------------');
272
                            console.log(ohsomeResults.data.substring(0, 500) + '...');
273
274
                            console.log('-----------------------------------------');
                        }
275
                        targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps);
276
277
278
279

                    }
                    console.timeEnd('convert');

280
                    //remove features where value = 0 or ratio = 0
281
                    if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues) {
282

283
                        console.log('Remove Zeros');
284
285
286
287
288
289
290
291
292
293
294

                        if(!this.isRatio){
                            targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection, 'value', 0);
                        } else {
                            targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValues(targetFeatureCollection, {
                                'value':0,
                                'value2': 0,
                                'ratio': null
                            });
                        }

295
296
297
                    }

                    console.time('computeValuePerArea');
298
                    if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea && !this.isRatio) {
299
300
301
302
303
304
305
306
307
308

                        targetFeatureCollection.features.forEach(
                            // @ts-ignore
                            (feature) => feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
                        );
                    }
                    console.timeEnd('computeValuePerArea');

                    await this.targetFeatureType.writeFeatures(targetFeatureCollection);

309
310
311
312
313
314
315
316
317
                    //update cursor for the next round of fetches
                    // @ts-ignore
                    cursor = sourceFeatureCollection.features[sourceFeatureCollection.features.length - 1].properties.id;
                    //update finished featureCount
                    this.currentFeatureCount += featureCount;

                    console.log('-----------------------------------------' + new Date().toISOString() + '-----------------------------------------');


318
319
                }
            } catch (e) {
320
                // console.log(e);
321
322
                this.sourceFeatureType.finalize();
                this.targetFeatureType.finalize();
323
                throw new Ohsome2XError('Could not create ohsome data.', {cursor: cursor}, e);
324
            }
325
326

        } else {
327
328
            // create all at once
            try {
329
330
                this.emit('progress', {type:'progress', processed: this.currentFeatureCount, total: this.totalFeatureCount, cursor: this.cursor, fetchSize: this.totalFeatureCount, timestamp: new Date().toISOString()});

331
332
333
                let targetFeatureCollection;
                const sourceFeatureCollection = await this.sourceFeatureType.getFeatures();

334
335
                this.currentFeatureCount = sourceFeatureCollection.features.length;

336
337
338
339
                console.time('computeArea');
                let idAreaMap: Map<any, number>;
                //in horizontal timestamp layout we can't have value and value_per_area, only one value is possible
                if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea) {
340
                    let idArea: any[any][number] = sourceFeatureCollection.features.map((feature: any) => [feature.properties.id, turfArea(feature.geometry)]);
341
342
343
344
345
346
347
348
349
350
351
352
353
                    idAreaMap = new Map(idArea);
                }
                console.timeEnd('computeArea');

                //build bpolys and add to ohsomeQuery
                //api requires ids to be strings
                //sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
                this.config.ohsomeQuery.bpolys = sourceFeatureCollection;

                console.time('query');
                const ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
                console.timeEnd('query');

354
355
356
357
358
359
360
361
362
363
                // temporary solution until ohsome-api changes
                this.harmonizeResponseStructure(ohsomeResults);

                let responseFormat = this.checkResponseFormat (ohsomeResults.data);
                this.isContributionView = responseFormat.isContributionView;
                this.isRatio = responseFormat.isRatio;

                if(responseFormat.isRatio){
                    this.harmonizeRatio_NaN_and_Infinity(ohsomeResults)
                }
364
365
366
367
368
369

                console.log('Start conversion ohsomeJSON or csv to GeoJSON');
                console.time('convert');
                if (shouldCreateGeometry) {
                    if (typeof ohsomeResults.data == 'string') { //if format is csv show beginning of result
                        console.log('-----------------------------------------');
370
                        console.log(ohsomeResults.data.substring(0, 400) + '...');
371
372
                        console.log('-----------------------------------------');
                    }
373
                    targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);
374
375
376
377
378

                } else {

                    if (typeof ohsomeResults.data == 'string') { //if format is csv show beginning of result
                        console.log('-----------------------------------------');
379
                        console.log(ohsomeResults.data.substring(0, 400) + '...');
380
381
                        console.log('-----------------------------------------');
                    }
382
                    targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, this.sourceIdType, shouldWriteHorizontalTimestamps);
383
384
385
386

                }
                console.timeEnd('convert');

387
                //remove features where value = 0 or ratio = NaN or null
388
389
                if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues) {

390
391
392
393
394
395
396
                    console.log('Remove Zeros...');

                    if(!this.isRatio){
                        targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection, 'value', 0);
                    } else {
                        targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValues(targetFeatureCollection, {'value':0, value2:0, 'ratio': null});
                    }
397
398
399
                }


400
401
                if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea && !this.isRatio) {
                    console.time('computeValuePerArea');
402
403
404
405
                    targetFeatureCollection.features.forEach(
                        // @ts-ignore
                        (feature) => feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
                    );
406
                    console.timeEnd('computeValuePerArea');
407
                }
408

409
                // console.log(JSON.stringify(targetFeatureCollection, undefined, 2));
410
411
                await this.targetFeatureType.writeFeatures(targetFeatureCollection);

412
                this.emit('progress', {type:'progress', processed: this.currentFeatureCount, total: this.totalFeatureCount, cursor: this.cursor, fetchSize: this.totalFeatureCount, timestamp: new Date().toISOString()});
413
            } catch (e) {
414
                // console.log(e);
415
416
                this.sourceFeatureType.finalize();
                this.targetFeatureType.finalize();
417
                throw new Ohsome2XError('Could not create ohsome data.', {cursor: this.cursor}, e);
418
            }
419
420

        }
421

422
423
424
425
426
        //createIndexes?
        try {
            // if(this.targetFeatureType.store.type == 'postgis' && !!this.config.target.createIndexes){
            if (this.targetFeatureType instanceof PgFeatureType) {
                this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
427
                let pgTarget = <TargetPostgisFeatureTypeConfig>this.config.target;
428
429
430
431
432
433
434
435
                if (!!pgTarget.createIndexes) {
                    await this.targetFeatureType.createIndex('id');
                    if (!!this.config.target.horizontalTimestampColumns) {
                        // TODO:horizontal
                        console.log('create indexes for horizontalTimestampColumns not yet implemented!');
                    } else {
                        // vertical: one timestamp column
                        if (this.isContributionView){
436

437
                            await this.targetFeatureType.createIndex('from_timestamp');
438

439
                            await this.targetFeatureType.clusterTable('from_timestamp');
440

441
                            await this.targetFeatureType.createIndex('to_timestamp');
442

443
                        } else {
444

445
                            await this.targetFeatureType.createIndex('timestamp');
446

447
448
                            await this.targetFeatureType.clusterTable('timestamp');
                        }
449

450

451
                        await this.targetFeatureType.createIndex('value');
452

453
454
455
456
457
458
                        if(this.isRatio){
                            await this.targetFeatureType.createIndex('value2');
                            await this.targetFeatureType.createIndex('ratio');
                        }

                        if (this.computeValuePerArea && !this.isRatio) {
459
460
461
                            await this.targetFeatureType.createIndex('value_per_area');
                        }
                    }
462

463
464
465
                    if (!!this.config.target.createGeometry) {
                        await this.targetFeatureType.createIndex('geom', 'gist');
                    }
466

467
468
469
470
                    await this.targetFeatureType.analyzeTable();
                }
            }
        } catch (e) {
471
            // console.log(e);
472
473
            this.sourceFeatureType.finalize();
            this.targetFeatureType.finalize();
474
            throw new Ohsome2XError('Could not create indexes.', {}, e);
475
476
        }

477
        //finalize e.g. close connections
478
479
480
        this.sourceFeatureType.finalize();
        this.targetFeatureType.finalize();

481
482
483
484
485
486
487
488
489
490
        this.log_end = new Date();
        console.log('Finished at: ' + this.log_end.toLocaleString());
        let log_diff = this.log_end.valueOf() - this.log_start.valueOf();
        let diffDays = Math.floor(log_diff / 86400000); // days
        let diffHrs = Math.floor((log_diff % 86400000) / 3600000); // hours
        let diffMins = Math.round(((log_diff % 86400000) % 3600000) / 60000); // minutes
        let diffSecs = Math.round(((log_diff % 86400000) % 3600000) % 60000 / 1000); // seconds
        console.log('====================================================================================');
        console.log(`Duration: ${diffDays} days ${diffHrs} hours ${diffMins} min ${diffSecs} seconds`);
        console.log('====================================================================================');
491

492
        this.emit('finished', {type: 'finished', duration:`${diffDays} days ${diffHrs} hours ${diffMins} min ${diffSecs} seconds`, duration_ms: log_diff, timestamp: this.log_end.toISOString()});
493
494
    }

495
///////////// methods
496
497


498
    async getOhsomeResults(ohsomeQuery: OhsomeQueryConfig | OhsomeQueryRatioConfig): Promise<AxiosResponse> {
499

500
        const isRatio = ohsomeQuery.queryType.includes("/ratio");
501
        // set empty strings to null
502
        const filter = (ohsomeQuery.filter != null && ohsomeQuery.filter.trim() != '')? ohsomeQuery.filter.trim() : undefined;
503
504
        // for ratio requests
        const filter2 = (isRatio && ohsomeQuery.filter2 != null && ohsomeQuery.filter2.trim() != '')? ohsomeQuery.filter2.trim() : undefined;
505
506
        const format: OhsomeQueryConfigFormat = (ohsomeQuery.format != null && ohsomeQuery.format.trim() != '')? ohsomeQuery.format.trim() as 'csv'|'json' : undefined;

507
        try {
508

509
510
511
            if (ohsomeQuery.bpolys == null) {
                throw Error('bpolys undefined in OhsomeQueryConfig');
            }
512
513
514
515
            let bpolyString = (typeof ohsomeQuery.bpolys === 'object')? JSON.stringify(ohsomeQuery.bpolys) : ohsomeQuery.bpolys.trim();

            //default query object, empty values will be removed due to issue with ohsomeAPI: https://gitlab.gistools.geog.uni-heidelberg.de/giscience/big-data/ohsome/ohsome-api/issues/72
            let dataObject: {} = {
516
                bpolys: bpolyString,
517
                filter: filter,
518
                filter2: filter2,
519
520
521
                time: ohsomeQuery.time,
                showMetadata: true,
                format: format
522
523
            }
            // removes properties with null or undefined
524
            const cleanDataObject: {[p: string]: string} = Object.entries(dataObject).reduce((a,[k,v]) => (v == null ? a : {...a, [k]:v}), {});
525

526
            let dataString = querystring.stringify(cleanDataObject);
527
528
529
530
531
532

            // always make a group by boundary query for backward compatibility
            // complete ohsomeAPI resource path should be used
            let queryType = ohsomeQuery.queryType.replace(/(.*)(\/)?(\/groupBy\/boundary)(\/)?/, "$1");

            console.log(`Querying ohsome-API: ${this.ohsomeApiUrl}/${queryType}/groupBy/boundary`);
533
534

            console.log('with params:', Object.entries(cleanDataObject).reduce((a,[k,v])=>({...a, [k]: (typeof v =='string')?v.substring(0,200): v }),{}));
535
            const stats = await axios(<AxiosRequestConfig>{
536
                url: `${this.ohsomeApiUrl}/${queryType}/groupBy/boundary`,
537
538
539
540
541
                method: 'post',
                header: {'content-type': 'application/x-www-form-urlencoded'},
                maxContentLength: 1024 * 1024 * 1024 * 1024,
                data: dataString
            });
542
543
544
545
546
547
            //only for json requests with showMetadata=true, not csv
            if (typeof stats.data === 'object' && "metadata" in stats.data){
                console.log('----------------------------------');
                console.log('Response Metadata', JSON.stringify(stats.data.metadata));
                console.log('----------------------------------');
            }
548
549
550
            return stats;
        } catch (e) {
            console.log('Ohsome API request failed.');
551
552
            this.axiosErrorPrinter(e);
            throw e; //Propagate the axios error
553
554
555
556
        }

    }

557
    checkResponseFormat(ohsomeGroupByBoundaryResponse: any): { isCSV:boolean; isGroupBy: boolean; isContributionView: boolean; isRatio: boolean } {
558

559
        const isCSV = (typeof ohsomeGroupByBoundaryResponse == 'string');
560

561
562
        let isContributionView: boolean;
        let isRatio: boolean;
563

564
565
566
567
568
569
570
571
572
573
        if (isCSV) {

            const lines = ohsomeGroupByBoundaryResponse.split("\n");

            //find first non-comment line
            for (let i = 0; i < lines.length; i++) {
                if (!lines[i].trimLeft().startsWith("#")) {
                    // check columnheaders for contributionview column names
                    const columns = lines[i].trim().split(";");
                    isContributionView = columns.includes("fromTimestamp") && columns.includes("toTimestamp");
574
575
                    // check isRatio: for non-grouped columnName is "ratio", for grouped response columnName is <someId>_ratio
                    isRatio = columns.includes("ratio") || columns.some((e: string)=> e.includes("_ratio"));
576
577
578
579
580
581
                    break;
                }
            }
        } else {
            //JSON Response

582
            // check isContributionView
583
584
585
            const hasFromTimestamps = ohsomeGroupByBoundaryResponse.groupByResult[0].result[0].fromTimestamp !== undefined;
            const hasToTimestamps = ohsomeGroupByBoundaryResponse.groupByResult[0].result[0].toTimestamp !== undefined;
            isContributionView = hasFromTimestamps && hasToTimestamps;
586
587
588

            // check isRatio
            isRatio = ohsomeGroupByBoundaryResponse.groupByResult[0].result[0].ratio !== undefined;
589
        }
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
        // return isContributionView;

        return {
            isCSV: isCSV,
            isGroupBy: true,
            isContributionView: isContributionView!,
            isRatio: isRatio!
        }
    }

    harmonizeResponseStructure(ohsomeResults: AxiosResponse): AxiosResponse {
        // HARMONIZE result structure as long as api does not change see: https://github.com/GIScience/ohsome-api/issues/16
        if (this.config.ohsomeQuery.format === 'json'){

            //plain ungrouped result: rename ratioResult into result
            if (ohsomeResults.data.ratioResult){
                delete Object.assign(ohsomeResults.data, {['result']: ohsomeResults.data['ratioResult'] })['ratioResult'];
            }

            //RatioGroupByBoundaryResponse into standard groupByResponse
            else if(ohsomeResults.data.groupByBoundaryResult){
                delete Object.assign(ohsomeResults.data, {['groupByResult']: ohsomeResults.data['groupByBoundaryResult'] })['groupByBoundaryResult'];

                // grouped ratio results
                if(ohsomeResults.data.groupByResult && ohsomeResults.data.groupByResult[0].ratioResult){
                    ohsomeResults.data.groupByResult.forEach((e: any)=>{
                        delete Object.assign(e, {['result']: e['ratioResult'] })['ratioResult'];
                    })
                }
            }
        }

        // end of response harmonization
        return ohsomeResults;
    }

    harmonizeRatio_NaN_and_Infinity(ohsomeResults: AxiosResponse): AxiosResponse {
        //Harmonize "NaN" strings of ratio to proper NULL types
       let isCSV = typeof ohsomeResults.data == 'string';

       //if it is CSV it is handled by a PapaParse config transform function in GeoJSONFeatureType.fromOhsome()
       if (!isCSV) {
           //json
           ohsomeResults.data.groupByResult.forEach((item: any)=>{
               item.result.forEach((tvv2r: any) => {
                   if (tvv2r.ratio === 'NaN' || tvv2r.ratio === 'Infinity'){tvv2r.ratio = null;}
               })
           })
       }
       return ohsomeResults;
640
641
    }

642
    axiosErrorPrinter(error: AxiosError) {
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
        if (error.response) {
            // The request was made and the server responded with a status code
            // that falls out of the range of 2xx
            console.log('Response error status: ' + error.response.status);
            console.log('Response error headers: ' + JSON.stringify(error.response.headers, null, 2));
            console.log('Response error data: ' + JSON.stringify(error.response.data, null, 2));
        } else if (error.request) {
            // The request was made but no response was received
            // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
            // http.ClientRequest in node.js
            console.log('No response received, request.');
        } else {
            // Something happened in setting up the request that triggered an Error
            console.log('Error', error.message);
        }
        delete error.config.data; //avoid bpoly logging
        console.log('Request config:\n', error.config);
660
    }
661
662
663
664
665
666
667

    onError(error: any) {
        console.log(error);
        if (this.sourceFeatureType && this.targetFeatureType) {
            this.sourceFeatureType.finalize();
            this.targetFeatureType.finalize();
        }
668
    }
669
670
671

}

672
673
export = Ohsome2X;
// export * from './config_types_interfaces';