Ohsome2X.ts 22.1 KB
Newer Older
1
2
import {
    Ohsome2XConfig,
3
    OhsomeQueryConfig, OhsomeQueryConfigFormat,
4
5
    TargetPostgisFeatureTypeConfig
} from './config_types_interfaces';
6
7
8
9
10
11
12
13
14
import normalizeUrl from 'normalize-url';
import {GeoJsonFeatureType} from './GeoJsonFeatureType';
import {PgFeatureType} from './PgFeatureType';
import {FeatureTypeFactory} from './FeatureTypeFactory';
import {Feature, Geometry} from '@turf/helpers';
import turfArea from '@turf/area';
import * as querystring from 'querystring';
import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';

15
import defaultConfig from './conf/default';
16

17
18
let OHSOME_API_URL = normalizeUrl(defaultConfig.OHSOME_API_URL); //remove trailing slash and other things

19
class Ohsome2X {
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
    private config: Ohsome2XConfig;
    private cursor: number | string | undefined;
    private fetchSize: number | null;
    private storeZeroValues: boolean;
    private computeValuePerArea: boolean | undefined;
    private sourceFeatureType: GeoJsonFeatureType | PgFeatureType | null;
    private targetFeatureType: GeoJsonFeatureType | PgFeatureType | null;
    private ohsomeApiUrl: string;
    private log_start: Date;
    private log_end: Date;
    private isContributionView: boolean | undefined;

    constructor(config: Ohsome2XConfig) {

        this.config = config;
35
36
37
        if ('cursor' in this.config.source) {
            this.cursor = this.config.source.cursor;
        }
38
39
40
41
42
43
44
45
        // @ts-ignore fetchSize only available for PostgisStoreConfig
        this.fetchSize = (!!this.config.source.fetchSize) ? parseInt(this.config.source.fetchSize) : null;
        this.storeZeroValues = (!('storeZeroValues' in this.config.target)) ? true : !!this.config.target.storeZeroValues;
        this.computeValuePerArea = this.config.target.computeValuePerArea;
        this.sourceFeatureType = null;
        this.targetFeatureType = null;

        this.ohsomeApiUrl = config.ohsomeApiUrl || OHSOME_API_URL;
46
47
48
49

        //default use csv because its way faster than parsing json
        this.config.ohsomeQuery.format = (this.config.ohsomeQuery.format == null || this.config.ohsomeQuery.format.trim() === '')? 'csv' : this.config.ohsomeQuery.format.trim() as 'csv' | 'json';

50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
        this.log_start = new Date();
        this.log_end = new Date();
        console.log('Start at: ' + this.log_start.toLocaleString());
        console.log('------------------------------');
        console.log('Start Ohsome2X with:');
        console.log('------------------------------');
        console.log('Ohsome-API:', this.ohsomeApiUrl);
        console.log(JSON.stringify(this.config.ohsomeQuery, null, 2));
        console.log('------------------------------');
        console.log('Source of statistical areas:');
        console.log(JSON.stringify(this.config.source, null, 2));
        console.log('------------------------------');
        console.log('Target of statistical results:');
        console.log(JSON.stringify(this.config.target, null, 2));
        console.log('------------------------------');
        //this.run().catch(console.log);
66
67
    }

68
69
70
71
72
73
74
75
76
77
    async run() {
        console.log('RUN');
        //initialize featureTypes with info if table aleady exists;
        try {
            this.sourceFeatureType = await FeatureTypeFactory.create(this.config.source);
            this.targetFeatureType = await FeatureTypeFactory.create(this.config.target);
        } catch (e) {
            console.log(e);
            throw new Error('Could not initialize FeatureTypes.');
        }
78

79
        //createGeometryOnTarget? default is true
80
        const shouldCreateGeometry = (this.config.target.createGeometry == null) ? true : this.config.target.createGeometry;
81
82
        // default is false
        const transformToWebmercator = (shouldCreateGeometry && !!this.config.target.transformToWebmercator);
83

84
85
        // write timestamp values in one or many columns? default is vertical (one column or two columns for contribution view queries)
        const shouldWriteHorizontalTimestamps = !!this.config.target.horizontalTimestampColumns;
86

87
        //dropTableIfexisits and views, do not delete if completing table from explicitly specified cursor
88
        // @ts-ignore
89
90
91
92
93
94
95
96
        if (this.targetFeatureType.store.type == 'postgis' && this.cursor == null) {
            await this.targetFeatureType.delete();
        }

        //createItertively or createAllAtOnce (posgis source and tagrget only)
        if (!(this.sourceFeatureType instanceof PgFeatureType && this.targetFeatureType instanceof PgFeatureType)) {
            //one or both of the stores are not postgisOnly => can't use fetch size, no chunkwise processing possible
            this.fetchSize = null;
97
        }
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203

        if (!!this.fetchSize) {
            // chunkwise processing
            this.sourceFeatureType = <PgFeatureType>this.sourceFeatureType;
            this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
            //create iteratively
            //check curser type: number or string?
            const cursorType = await this.sourceFeatureType.getIdJsTypeFromPg();
            console.log("cursorType", cursorType);

            let cursor;
            //cursor was defined in config
            if (this.cursor != null){
                cursor = this.cursor;
            } else if (cursorType === 'number'){
                cursor = Number.MIN_SAFE_INTEGER;
            } else {
                //cursorType == 'string'
                cursor = '';
            }

            let featureCount = 1; // 1 to pass break test first time
            try {
                while (true) {

                    const sourceFeatureCollection: any = await this.sourceFeatureType.getFeaturesByCursorAndLimit(cursor, this.fetchSize);
                    let targetFeatureCollection;

                    featureCount = sourceFeatureCollection.features.length;
                    if (featureCount == 0) {
                        console.log('No more cells.');
                        break;
                    }
                    // @ts-ignore
                    //cursor = parseInt(sourceFeatureCollection.features[sourceFeatureCollection.features.length - 1].properties.id);
                    cursor = sourceFeatureCollection.features[sourceFeatureCollection.features.length - 1].properties.id;

                    console.time('computeArea');
                    let idAreaMap: Map<any, number>;
                    if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea) {
                        // @ts-ignore
                        let idArea: [any, number][] = sourceFeatureCollection.features.map((feature: Feature) => [feature.properties.id, turfArea(<Geometry>feature.geometry)]);
                        idAreaMap = new Map(idArea);
                    }
                    console.timeEnd('computeArea');

                    //build bpolys and add to ohsomeQuery
                    //api requires ids to be strings
                    //sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
                    this.config.ohsomeQuery.bpolys = sourceFeatureCollection;


                    console.time('query');
                    let ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
                    console.timeEnd('query');

                    this.isContributionView = this.checkIsContributionView(ohsomeResults.data);

                    console.log('Start conversion ohsomeJSON to GeoJSON');
                    console.time('convert');
                    if (shouldCreateGeometry) {
                        if (typeof ohsomeResults.data == 'string') {
                            console.log('-----------------------------------------');
                            console.log(ohsomeResults.data.substring(0, 400));
                            console.log('-----------------------------------------');
                        }
                        targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);

                    } else {
                        if (typeof ohsomeResults.data == 'string') {
                            console.log('-----------------------------------------');
                            console.log(ohsomeResults.data.substring(0, 400));
                            console.log('-----------------------------------------');
                        }
                        targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);

                    }
                    console.timeEnd('convert');

                    if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues) {
                        //remove features where value = 0
                        console.log('Remove Zeros');
                        console.time('removeZero');
                        targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection, 'value', 0);
                        console.timeEnd('removeZero');
                    }

                    console.time('computeValuePerArea');
                    if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea) {

                        targetFeatureCollection.features.forEach(
                            // @ts-ignore
                            (feature) => feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
                        );
                    }
                    console.timeEnd('computeValuePerArea');

                    await this.targetFeatureType.writeFeatures(targetFeatureCollection);
                    //await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);

                }
            } catch (e) {
                console.log(e);
                this.sourceFeatureType.finalize();
                this.targetFeatureType.finalize();
                throw new Error('Could not create ohsome data.')
204
            }
205
206

        } else {
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
            // create all at once
            try {
                let targetFeatureCollection;
                const sourceFeatureCollection = await this.sourceFeatureType.getFeatures();

                console.time('computeArea');
                let idAreaMap: Map<any, number>;
                //in horizontal timestamp layout we can't have value and value_per_area, only one value is possible
                if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea) {
                    let idArea = sourceFeatureCollection.features.map((feature: any) => [feature.properties.id, turfArea(feature.geometry)]);
                    idAreaMap = new Map(idArea);
                }
                console.timeEnd('computeArea');

                //build bpolys and add to ohsomeQuery
                //api requires ids to be strings
                //sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
                this.config.ohsomeQuery.bpolys = sourceFeatureCollection;

                console.time('query');
                const ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
                console.timeEnd('query');

                this.isContributionView = this.checkIsContributionView(ohsomeResults.data);

                console.log('Start conversion ohsomeJSON or csv to GeoJSON');
                console.time('convert');
                if (shouldCreateGeometry) {
                    if (typeof ohsomeResults.data == 'string') { //if format is csv show beginning of result
                        console.log('-----------------------------------------');
                        console.log(ohsomeResults.data.substring(0, 400));
                        console.log('-----------------------------------------');
                    }
                    targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);

                } else {

                    if (typeof ohsomeResults.data == 'string') { //if format is csv show beginning of result
                        console.log('-----------------------------------------');
                        console.log(ohsomeResults.data.substring(0, 400));
                        console.log('-----------------------------------------');
                    }
                    targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);

                }
                console.timeEnd('convert');

                if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues) {
                    //remove features where value = 0

                    targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection, 'value', 0);
                }

                console.time('computeValuePerArea');
                if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea) {

                    targetFeatureCollection.features.forEach(
                        // @ts-ignore
                        (feature) => feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
                    );
                }
                console.timeEnd('computeValuePerArea');
269
                // console.log(JSON.stringify(targetFeatureCollection, undefined, 2));
270
271
272
273
274
275
276
277
                await this.targetFeatureType.writeFeatures(targetFeatureCollection);

                //await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);
            } catch (e) {
                console.log(e);
                this.sourceFeatureType.finalize();
                this.targetFeatureType.finalize();
                throw new Error('Could not create ohsome data.');
278
            }
279
280

        }
281

282
283
284
285
286
        //createIndexes?
        try {
            // if(this.targetFeatureType.store.type == 'postgis' && !!this.config.target.createIndexes){
            if (this.targetFeatureType instanceof PgFeatureType) {
                this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
287
                let pgTarget = <TargetPostgisFeatureTypeConfig>this.config.target;
288
289
290
291
292
293
294
295
                if (!!pgTarget.createIndexes) {
                    await this.targetFeatureType.createIndex('id');
                    if (!!this.config.target.horizontalTimestampColumns) {
                        // TODO:horizontal
                        console.log('create indexes for horizontalTimestampColumns not yet implemented!');
                    } else {
                        // vertical: one timestamp column
                        if (this.isContributionView){
296

297
                            await this.targetFeatureType.createIndex('from_timestamp');
298

299
                            await this.targetFeatureType.clusterTable('from_timestamp');
300

301
                            await this.targetFeatureType.createIndex('to_timestamp');
302

303
                        } else {
304

305
                            await this.targetFeatureType.createIndex('timestamp');
306

307
308
                            await this.targetFeatureType.clusterTable('timestamp');
                        }
309

310
                        await this.targetFeatureType.createIndex('value');
311

312
313
314
315
                        if (this.computeValuePerArea) {
                            await this.targetFeatureType.createIndex('value_per_area');
                        }
                    }
316

317
318
319
                    if (!!this.config.target.createGeometry) {
                        await this.targetFeatureType.createIndex('geom', 'gist');
                    }
320

321
322
323
324
325
326
327
328
                    await this.targetFeatureType.analyzeTable();
                }
            }
        } catch (e) {
            console.log(e);
            this.sourceFeatureType.finalize();
            this.targetFeatureType.finalize();
            throw new Error('Could not create indexes.');
329
330
        }

331
        //finalize e.g. close connections
332
333
334
        this.sourceFeatureType.finalize();
        this.targetFeatureType.finalize();

335
336
337
338
339
340
341
342
343
344
        this.log_end = new Date();
        console.log('Finished at: ' + this.log_end.toLocaleString());
        let log_diff = this.log_end.valueOf() - this.log_start.valueOf();
        let diffDays = Math.floor(log_diff / 86400000); // days
        let diffHrs = Math.floor((log_diff % 86400000) / 3600000); // hours
        let diffMins = Math.round(((log_diff % 86400000) % 3600000) / 60000); // minutes
        let diffSecs = Math.round(((log_diff % 86400000) % 3600000) % 60000 / 1000); // seconds
        console.log('====================================================================================');
        console.log(`Duration: ${diffDays} days ${diffHrs} hours ${diffMins} min ${diffSecs} seconds`);
        console.log('====================================================================================');
345
346
    }

347
///////////// methods
348
349


350
    async getOhsomeResults(ohsomeQuery: OhsomeQueryConfig): Promise<AxiosResponse> {
351
352
353
354
355
356
357
358

        // set empty strings to null
        const keys   = (ohsomeQuery.keys != null && ohsomeQuery.keys.trim() != '')? ohsomeQuery.keys.trim() : undefined;
        const values = (ohsomeQuery.values != null && ohsomeQuery.values.trim() != '')? ohsomeQuery.values.trim() : undefined;
        const format: OhsomeQueryConfigFormat = (ohsomeQuery.format != null && ohsomeQuery.format.trim() != '')? ohsomeQuery.format.trim() as 'csv'|'json' : undefined;



359
        try {
360

361
362
363
            if (ohsomeQuery.bpolys == null) {
                throw Error('bpolys undefined in OhsomeQueryConfig');
            }
364
365
366
367
            let bpolyString = (typeof ohsomeQuery.bpolys === 'object')? JSON.stringify(ohsomeQuery.bpolys) : ohsomeQuery.bpolys.trim();

            //default query object, empty values will be removed due to issue with ohsomeAPI: https://gitlab.gistools.geog.uni-heidelberg.de/giscience/big-data/ohsome/ohsome-api/issues/72
            let dataObject: {} = {
368
369
370
371
372
373
374
                bpolys: bpolyString,
                keys: keys,
                values: values,
                time: ohsomeQuery.time,
                types: ohsomeQuery.types,
                showMetadata: true,
                format: format
375
376
            }
            // removes properties with null or undefined
377
            const cleanDataObject: {[p: string]: string} = Object.entries(dataObject).reduce((a,[k,v]) => (v == null ? a : {...a, [k]:v}), {});
378

379
            let dataString = querystring.stringify(cleanDataObject);
380
381
382
383
384
385

            // always make a group by boundary query for backward compatibility
            // complete ohsomeAPI resource path should be used
            let queryType = ohsomeQuery.queryType.replace(/(.*)(\/)?(\/groupBy\/boundary)(\/)?/, "$1");

            console.log(`Querying ohsome-API: ${this.ohsomeApiUrl}/${queryType}/groupBy/boundary`);
386
387

            console.log('with params:', Object.entries(cleanDataObject).reduce((a,[k,v])=>({...a, [k]: (typeof v =='string')?v.substring(0,200): v }),{}));
388
            const stats = await axios(<AxiosRequestConfig>{
389
                url: `${this.ohsomeApiUrl}/${queryType}/groupBy/boundary`,
390
391
392
393
394
                method: 'post',
                header: {'content-type': 'application/x-www-form-urlencoded'},
                maxContentLength: 1024 * 1024 * 1024 * 1024,
                data: dataString
            });
395
396
397
398
399
400
            //only for json requests with showMetadata=true, not csv
            if (typeof stats.data === 'object' && "metadata" in stats.data){
                console.log('----------------------------------');
                console.log('Response Metadata', JSON.stringify(stats.data.metadata));
                console.log('----------------------------------');
            }
401
402
403
404
405
            return stats;
        } catch (e) {
            console.log('Ohsome API request failed.');
            this.axiosErrorHandler(e);
            throw new Error(e.message);
406
407
408
409
        }

    }

410
    checkIsContributionView(ohsomeGroupByBoundaryResponse: any): boolean {
411

412
        const isCSV = (typeof ohsomeGroupByBoundaryResponse == 'string');
413

414
        let isContributionView: boolean = true;
415

416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
        if (isCSV) {

            const lines = ohsomeGroupByBoundaryResponse.split("\n");

            //find first non-comment line
            for (let i = 0; i < lines.length; i++) {
                if (!lines[i].trimLeft().startsWith("#")) {
                    // check columnheaders for contributionview column names
                    const columns = lines[i].trim().split(";");
                    isContributionView = columns.includes("fromTimestamp") && columns.includes("toTimestamp");
                    break;
                }
            }
        } else {
            //JSON Response

            const hasFromTimestamps = ohsomeGroupByBoundaryResponse.groupByResult[0].result[0].fromTimestamp !== undefined;
            const hasToTimestamps = ohsomeGroupByBoundaryResponse.groupByResult[0].result[0].toTimestamp !== undefined;
            isContributionView = hasFromTimestamps && hasToTimestamps;
        }
        return isContributionView;
437
438
    }

439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
    axiosErrorHandler(error: any) {
        if (error.response) {
            // The request was made and the server responded with a status code
            // that falls out of the range of 2xx
            console.log('Response error status: ' + error.response.status);
            console.log('Response error headers: ' + JSON.stringify(error.response.headers, null, 2));
            console.log('Response error data: ' + JSON.stringify(error.response.data, null, 2));
        } else if (error.request) {
            // The request was made but no response was received
            // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
            // http.ClientRequest in node.js
            console.log('No response received, request.');
        } else {
            // Something happened in setting up the request that triggered an Error
            console.log('Error', error.message);
        }
        delete error.config.data; //avoid bpoly logging
        console.log('Request config:\n', error.config);
457
    }
458
459
460
461
462
463
464

    onError(error: any) {
        console.log(error);
        if (this.sourceFeatureType && this.targetFeatureType) {
            this.sourceFeatureType.finalize();
            this.targetFeatureType.finalize();
        }
465
    }
466
467
468

}

469
470
export = Ohsome2X;
// export * from './config_types_interfaces';