Ohsome2X.ts 17.5 KB
Newer Older
1

2
// const FeatureType = require('./FeatureType.js');
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
// const PgFeatureType = require('./PgFeatureType.js');
// const querystring = require('querystring');
// const turfHelpers = require('@turf/helpers');
// const turfArea = require('@turf/area');
// const axios = require('axios');
// const normalizeUrl = require('normalize-url');
// const FeatureTypeFactory = require('./FeatureTypeFactory.js');
// const GeoJsonFeatureType = require('./GeoJsonFeatureType.js');
import {Ohsome2XConfig, PostgisFeatureTypeConfig, OhsomeQueryConfig} from './config_types_interfaces';
import normalizeUrl from 'normalize-url';
import {GeoJsonFeatureType} from './GeoJsonFeatureType';
import {PgFeatureType} from './PgFeatureType';
import {FeatureTypeFactory} from './FeatureTypeFactory';
import {Feature, Geometry} from '@turf/helpers';
import turfArea from '@turf/area';
import * as querystring from 'querystring';
import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';

21
22
23
const defaultConfig = require('./conf/default.js');
let OHSOME_API_URL = normalizeUrl(defaultConfig.OHSOME_API_URL); //remove trailing slash and other things

24
25
26
27
28
29
30
31
32
33
34
export class Ohsome2X {
  private config: Ohsome2XConfig;
  private cursor: number;
  private fetchSize: number | null;
  private storeZeroValues: boolean;
  private computeValuePerArea: boolean | undefined;
  private sourceFeatureType: GeoJsonFeatureType | PgFeatureType | null;
  private targetFeatureType: GeoJsonFeatureType | PgFeatureType | null;
  private ohsomeApiUrl: string;
  private log_start: Date;
  private log_end: Date;
35

36
  constructor(config : Ohsome2XConfig){
37
38
39
40
41

    // let ohsomeQuery = {
    //   queryType: 'elements/count',
    //   keys: ['natural'].join(),
    //   values: ['tree'].join(),
42
    //   @types: ['node'],
43
44
45
46
    //   time: '2014-01-01/2017-01-01/P1Y'
    // }
    //
    // let store =  {
47
    //   type: 'postgis', //postgis,
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
    //   host: 'example.com',
    //   port: 5432,
    //   user: 'foo',
    //   password: '*****',
    //   database: 'your-db-name',
    //   //schema:
    // }
    //
    // let sourceFeatureType = {
    //   name:'isea3h_nepal_res10_verkl', //postgis: tableName, geojson: optional name
    //   geometryId: 'id', //postgisOnly //unique not null, will be sorted when using fetchSize
    //   geometryColumn: 'geom', //postgisOnly //must be in EPSG:4326
    //   store: store, //store eg. postgis, geoJSON<FeatureCollection>,...
    //   fetchSize: 0 //postgisOnly
    // }
    //
    // let targetFeatureType = {
    //   name: ohsomeQuery.keys[0] + '_' +ohsomeQuery.values[0],
    //   store: store,
    //   horizontalTimestampColumns: false,
    //   createGeometry: false, //TODO default should be true
    // }
    //
    //
    // //let
    // config = {
    //   source: sourceFeatureType,
    //   target: targetFeatureType,
    //   ohsomeQuery: ohsomeQuery
    // }

    this.config = config;
    this.cursor = this.config.source.cursor || 0;
81
    // @ts-ignore fetchSize only available for PostgisStoreConfig
82
83
84
    this.fetchSize = (!!this.config.source.fetchSize)? parseInt(this.config.source.fetchSize) : null;
    this.storeZeroValues = ( !('storeZeroValues' in this.config.target) )? true : !!this.config.target.storeZeroValues;
    this.computeValuePerArea = this.config.target.computeValuePerArea;
85
86
87
    this.sourceFeatureType = null;
    this.targetFeatureType = null;

88
89
90

    this.ohsomeApiUrl = config.ohsomeApiUrl || OHSOME_API_URL;
    this.log_start = new Date();
91
    this.log_end = new Date();
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
    console.log('Start at: ' + this.log_start.toLocaleString());
    console.log('------------------------------');
    console.log('Start Ohsome2X with:');
    console.log('------------------------------');
    console.log('Ohsome-API:', this.ohsomeApiUrl);
    console.log(JSON.stringify(this.config.ohsomeQuery,null, 2));
    console.log('------------------------------');
    console.log('Source of statistical areas:');
    console.log(JSON.stringify(this.config.source,null, 2));
    console.log('------------------------------');
    console.log('Target of statistical results:');
    console.log(JSON.stringify(this.config.target,null, 2));
    console.log('------------------------------');
    //this.run().catch(console.log);
  }

  async run(){
    console.log('RUN');
    //initialize featureTypes with info if table aleady exists;
    try {
      this.sourceFeatureType = await FeatureTypeFactory.create(this.config.source);
      this.targetFeatureType = await FeatureTypeFactory.create(this.config.target);
    } catch (e) {
      console.log(e);
      throw new Error('Could not initialize FeatureTypes.');
    }

    //createGeometryOnTarget? default is true
    const shouldCreateGeometry = (this.config.target.createGeometry == null)? true: !!this.config.target.createGeometry;
    // default is false
    const transformToWebmercator = ( shouldCreateGeometry && !!this.config.target.transformToWebmercator);

    // write timestamp values in one or many columns? default is vertical (one column)
    const shouldWriteHorizontalTimestamps = !!this.config.target.horizontalTimestampColumns;

    //dropTableIfexisits and views, do not delete if completing table from cursor > 0
128
    // @ts-ignore
129
130
131
132
133
134
135
136
137
138
139
    if (this.targetFeatureType.store.type == 'postgis' && this.cursor == 0 ){
      await this.targetFeatureType.delete();
    }

    //createItertively or createAllAtOnce (posgis source and tagrget only)
    if (! (this.sourceFeatureType instanceof PgFeatureType && this.targetFeatureType instanceof PgFeatureType) ) {
      //one or both of the stores are not postgisOnly
      this.fetchSize = null;
    }

    if (!!this.fetchSize){
140
141
      this.sourceFeatureType = <PgFeatureType>this.sourceFeatureType;
      this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
142
143
144
145
146
147
148
149
      //create iteratively
      let cursor = this.cursor;
      let featureCount = 1; // 1 to pass break test first time
  try {
      while (true) {

        const sourceFeatureCollection = await this.sourceFeatureType.getFeaturesByCursorAndLimit(cursor, this.fetchSize);
        let targetFeatureCollection;
150

151
152
153
154
155
        featureCount = sourceFeatureCollection.features.length;
        if(featureCount == 0){
          console.log('No more cells.');
          break;
        }
156
        // @ts-ignore
157
158
159
        cursor = parseInt(sourceFeatureCollection.features[sourceFeatureCollection.features.length-1].properties.id);

        console.time('computeArea');
160
        let idAreaMap: Map<any,number>;
161
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
162
163
          // @ts-ignore
          let idArea:[any,number][] = sourceFeatureCollection.features.map((feature: Feature)=>[feature.properties.id, turfArea(<Geometry>feature.geometry)]);
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
          idAreaMap = new Map(idArea);
        }
        console.timeEnd('computeArea');

        //build bpolys and add to ohsomeQuery
        //api requires ids to be strings
        //sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
        this.config.ohsomeQuery.bpolys = sourceFeatureCollection;


        console.time('query');
        let ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
        console.timeEnd('query');
        console.log('Start conversion ohsomeJSON to GeoJSON');
        console.time('convert');
        if (shouldCreateGeometry) {
            if (typeof ohsomeResults.data == 'string') {
              console.log('-----------------------------------------');
182
              console.log(ohsomeResults.data.substring(0,400));
183
              console.log('-----------------------------------------');
184
            }
185
186
187
188
189
            targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);

        } else {
            if (typeof ohsomeResults.data == 'string') {
              console.log('-----------------------------------------');
190
              console.log(ohsomeResults.data.substring(0,400));
191
              console.log('-----------------------------------------');
192
            }
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
            targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);

        }
        console.timeEnd('convert');

        if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues){
          //remove features where value = 0
         console.log('Remove Zeros');
          console.time('removeZero');
          targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection,'value', 0);
          console.timeEnd('removeZero');
        }

        console.time('computeValuePerArea');
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
208

209
          targetFeatureCollection.features.forEach(
210
              // @ts-ignore
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
              (feature)=>feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
          );
        }
        console.timeEnd('computeValuePerArea');

        await this.targetFeatureType.writeFeatures(targetFeatureCollection);
        //await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);

      }
    } catch (e) {
      console.log(e);
      this.sourceFeatureType.finalize();
      this.targetFeatureType.finalize();
      throw new Error('Could not create ohsome data.')
    }

    } else {
      // create all at once
      try {
        let targetFeatureCollection;
        const sourceFeatureCollection = await this.sourceFeatureType.getFeatures();

        console.time('computeArea');
234
        let idAreaMap: Map<any,number>;
235
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
236
          let idArea = sourceFeatureCollection.features.map((feature:any)=>[feature.properties.id, turfArea(feature.geometry)]);
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
          idAreaMap = new Map(idArea);
        }
        console.timeEnd('computeArea');

        //build bpolys and add to ohsomeQuery
        //api requires ids to be strings
        //sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
        this.config.ohsomeQuery.bpolys = sourceFeatureCollection;

        console.time('query');
        const  ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
        console.timeEnd('query');

        console.log('Start conversion ohsomeJSON or csv to GeoJSON');
        console.time('convert');
        if (shouldCreateGeometry) {
          if (typeof ohsomeResults.data == 'string') {
            console.log('-----------------------------------------');
255
            console.log(ohsomeResults.data.substring(0,400));
256
            console.log('-----------------------------------------');
257
          }
258
259
260
261
262
263
            targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);

        } else {

          if (typeof ohsomeResults.data == 'string') {
            console.log('-----------------------------------------');
264
            console.log(ohsomeResults.data.substring(0,400));
265
            console.log('-----------------------------------------');
266
          }
267
268
269
270
271
272
273
274
275
276
277
278
279
          targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);

        }
        console.timeEnd('convert');

        if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues){
          //remove features where value = 0

          targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection,'value', 0);
        }

        console.time('computeValuePerArea');
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
280

281
          targetFeatureCollection.features.forEach(
282
              // @ts-ignore
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
              (feature)=>feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
          );
        }
        console.timeEnd('computeValuePerArea');

        await this.targetFeatureType.writeFeatures(targetFeatureCollection);

        //await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);
      } catch (e) {
        console.log(e);
        this.sourceFeatureType.finalize();
        this.targetFeatureType.finalize();
        throw new Error('Could not create ohsome data.');
      }

    }

    //createIndexes?
    try {
302
303
304
305
306
      // if(this.targetFeatureType.store.type == 'postgis' && !!this.config.target.createIndexes){
        if(this.targetFeatureType instanceof PgFeatureType) {
          this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
          let pgTarget = <PostgisFeatureTypeConfig>this.config.target;
          if( !! pgTarget.createIndexes){
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
        await this.targetFeatureType.createIndex('id');
        if(!!this.config.target.horizontalTimestampColumns){
          // TODO:horizontal
          console.log('create indexes for horizontalTimestampColumns not yet implemented!');
        } else {
          // vertical: one timestamp column
          await this.targetFeatureType.createIndex('timestamp');

          await this.targetFeatureType.clusterTable('timestamp');

          await this.targetFeatureType.createIndex('value');

          if (this.computeValuePerArea) {
            await this.targetFeatureType.createIndex('value_per_area');
          }
        }

        if(!!this.config.target.createGeometry){
          await this.targetFeatureType.createIndex('geom', 'gist');
        }

        await this.targetFeatureType.analyzeTable();
329
      }}
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
    } catch (e) {
      console.log(e);
      this.sourceFeatureType.finalize();
      this.targetFeatureType.finalize();
      throw new Error('Could not create indexes.');
    }

    //finalize e.g. close connections
    this.sourceFeatureType.finalize();
    this.targetFeatureType.finalize();

    this.log_end = new Date();
    console.log('Finished at: ' + this.log_end.toLocaleString());
    let log_diff = this.log_end.valueOf() - this.log_start.valueOf();
    let diffDays = Math.floor(log_diff / 86400000); // days
    let diffHrs = Math.floor((log_diff % 86400000) / 3600000); // hours
    let diffMins = Math.round(((log_diff % 86400000) % 3600000) / 60000); // minutes
    let diffSecs = Math.round(((log_diff % 86400000) % 3600000) % 60000 / 1000 ); // seconds
    console.log('====================================================================================');
    console.log(`Duration: ${diffDays} days ${diffHrs} hours ${diffMins} min ${diffSecs} seconds` );
    console.log('====================================================================================');
  }

///////////// methods


356
  async getOhsomeResults(ohsomeQuery: OhsomeQueryConfig): Promise<AxiosResponse>{
357
358
359
360
361
362
363
364
365
366
367
//     axios.interceptors.request.use(request => {
//   console.log('Starting Request', request)
//   return request
// });
    const keys = ohsomeQuery.keys || '';
    const values = ohsomeQuery.values || '';
    const format = ohsomeQuery.format || 'json';
    try {
      console.log('----------------------------------');
      console.log('Stringify Cells');
      console.time('stringify');
368
369
370
      if(ohsomeQuery.bpolys == null){
        throw Error('bpolys undefined in OhsomeQueryConfig');
      }
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
      let bpolyString = JSON.stringify(ohsomeQuery.bpolys);
      console.timeEnd('stringify');
      console.log('----------------------------------');
      console.log('Stringify querystring');
      console.time('querystring');
      let dataString = querystring.stringify({
        bpolys: bpolyString,
        keys: keys,
        values: values,
        time: ohsomeQuery.time,
        types: ohsomeQuery.types,
        showMetadata: true,
        format: format
      });
      console.timeEnd('querystring');
      console.log('----------------------------------');
      console.log(`Querying ohsome-API: ${this.ohsomeApiUrl}/${ohsomeQuery.queryType}/groupBy/boundary`);
388
      const stats = await axios(<AxiosRequestConfig>{
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
          url: `${this.ohsomeApiUrl}/${ohsomeQuery.queryType}/groupBy/boundary`,
          method: 'post',
          header: {'content-type': 'application/x-www-form-urlencoded'},
          maxContentLength: 1024*1024*1024*1024,
          data: dataString
      });
      console.log('----------------------------------');
      console.log('Response Metadata', JSON.stringify(stats.data.metadata));
      console.log('----------------------------------');
      return stats;
    } catch (e) {
        console.log('Ohsome API request failed.');
        this.axiosErrorHandler(e);
        throw new Error(e.message);
    }

  }

407
  axiosErrorHandler(error:any){
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
    if (error.response) {
      // The request was made and the server responded with a status code
      // that falls out of the range of 2xx
        console.log('Response error status: ' + error.response.status);
        console.log('Response error headers: ' + JSON.stringify(error.response.headers,null,2));
        console.log('Response error data: ' + JSON.stringify(error.response.data,null,2));
    } else if (error.request) {
      // The request was made but no response was received
      // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
      // http.ClientRequest in node.js
      console.log('No response received, request.');
    } else {
      // Something happened in setting up the request that triggered an Error
      console.log('Error', error.message);
    }
    delete error.config.data; //avoid bpoly logging
    console.log('Request config:\n', error.config);
  }

427
428
429
430
431
432
   onError(error:any) {
    console.log(error);
    if (this.sourceFeatureType && this.targetFeatureType){
      this.sourceFeatureType.finalize();
      this.targetFeatureType.finalize();
    }
433
434
435
436
  }

}