Ohsome2X.ts

// const FeatureType = require('./FeatureType.js');
// const PgFeatureType = require('./PgFeatureType.js');
// const querystring = require('querystring');
// const turfHelpers = require('@turf/helpers');
// const turfArea = require('@turf/area');
// const axios = require('axios');
// const normalizeUrl = require('normalize-url');
// const FeatureTypeFactory = require('./FeatureTypeFactory.js');
// const GeoJsonFeatureType = require('./GeoJsonFeatureType.js');
import {Ohsome2XConfig, PostgisFeatureTypeConfig, OhsomeQueryConfig} from './config_types_interfaces';
import normalizeUrl from 'normalize-url';
import {GeoJsonFeatureType} from './GeoJsonFeatureType';
import {PgFeatureType} from './PgFeatureType';
import {FeatureTypeFactory} from './FeatureTypeFactory';
import {Feature, Geometry} from '@turf/helpers';
import turfArea from '@turf/area';
import * as querystring from 'querystring';
import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';

import defaultConfig from './conf/default';
let OHSOME_API_URL = normalizeUrl(defaultConfig.OHSOME_API_URL); //remove trailing slash and other things

class Ohsome2X {
  private config: Ohsome2XConfig;
  private cursor: number;
  private fetchSize: number | null;
  private storeZeroValues: boolean;
  private computeValuePerArea: boolean | undefined;
  private sourceFeatureType: GeoJsonFeatureType | PgFeatureType | null;
  private targetFeatureType: GeoJsonFeatureType | PgFeatureType | null;
  private ohsomeApiUrl: string;
  private log_start: Date;
  private log_end: Date;

  constructor(config : Ohsome2XConfig){

    // let ohsomeQuery = {
    //   queryType: 'elements/count',
    //   keys: ['natural'].join(),
    //   values: ['tree'].join(),
    //   @types: ['node'],
    //   time: '2014-01-01/2017-01-01/P1Y'
    // }
    //
    // let store =  {
    //   type: 'postgis', //postgis,
    //   host: 'example.com',
    //   port: 5432,
    //   user: 'foo',
    //   password: '*****',
    //   database: 'your-db-name',
    //   //schema:
    // }
    //
    // let sourceFeatureType = {
    //   name:'isea3h_nepal_res10_verkl', //postgis: tableName, geojson: optional name
    //   geometryId: 'id', //postgisOnly //unique not null, will be sorted when using fetchSize
    //   geometryColumn: 'geom', //postgisOnly //must be in EPSG:4326
    //   store: store, //store eg. postgis, geoJSON<FeatureCollection>,...
    //   fetchSize: 0 //postgisOnly
    // }
    //
    // let targetFeatureType = {
    //   name: ohsomeQuery.keys[0] + '_' +ohsomeQuery.values[0],
    //   store: store,
    //   horizontalTimestampColumns: false,
    //   createGeometry: false, //TODO default should be true
    // }
    //
    //
    // //let
    // config = {
    //   source: sourceFeatureType,
    //   target: targetFeatureType,
    //   ohsomeQuery: ohsomeQuery
    // }

    this.config = config;
    this.cursor = this.config.source.cursor || 0;
    // @ts-ignore fetchSize only available for PostgisStoreConfig
    this.fetchSize = (!!this.config.source.fetchSize)? parseInt(this.config.source.fetchSize) : null;
    this.storeZeroValues = ( !('storeZeroValues' in this.config.target) )? true : !!this.config.target.storeZeroValues;
    this.computeValuePerArea = this.config.target.computeValuePerArea;
    this.sourceFeatureType = null;
    this.targetFeatureType = null;


    this.ohsomeApiUrl = config.ohsomeApiUrl || OHSOME_API_URL;
    this.log_start = new Date();
    this.log_end = new Date();
    console.log('Start at: ' + this.log_start.toLocaleString());
    console.log('------------------------------');
    console.log('Start Ohsome2X with:');
    console.log('------------------------------');
    console.log('Ohsome-API:', this.ohsomeApiUrl);
    console.log(JSON.stringify(this.config.ohsomeQuery,null, 2));
    console.log('------------------------------');
    console.log('Source of statistical areas:');
    console.log(JSON.stringify(this.config.source,null, 2));
    console.log('------------------------------');
    console.log('Target of statistical results:');
    console.log(JSON.stringify(this.config.target,null, 2));
    console.log('------------------------------');
    //this.run().catch(console.log);
  }

  async run(){
    console.log('RUN');
    //initialize featureTypes with info if table aleady exists;
    try {
      this.sourceFeatureType = await FeatureTypeFactory.create(this.config.source);
      this.targetFeatureType = await FeatureTypeFactory.create(this.config.target);
    } catch (e) {
      console.log(e);
      throw new Error('Could not initialize FeatureTypes.');
    }

    //createGeometryOnTarget? default is true
    const shouldCreateGeometry = (this.config.target.createGeometry == null)? true: !!this.config.target.createGeometry;
    // default is false
    const transformToWebmercator = ( shouldCreateGeometry && !!this.config.target.transformToWebmercator);

    // write timestamp values in one or many columns? default is vertical (one column)
    const shouldWriteHorizontalTimestamps = !!this.config.target.horizontalTimestampColumns;

    //dropTableIfexisits and views, do not delete if completing table from cursor > 0
    // @ts-ignore
    if (this.targetFeatureType.store.type == 'postgis' && this.cursor == 0 ){
      await this.targetFeatureType.delete();
    }

    //createItertively or createAllAtOnce (posgis source and tagrget only)
    if (! (this.sourceFeatureType instanceof PgFeatureType && this.targetFeatureType instanceof PgFeatureType) ) {
      //one or both of the stores are not postgisOnly
      this.fetchSize = null;
    }

    if (!!this.fetchSize){
      this.sourceFeatureType = <PgFeatureType>this.sourceFeatureType;
      this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
      //create iteratively
      let cursor = this.cursor;
      let featureCount = 1; // 1 to pass break test first time
  try {
      while (true) {

        const sourceFeatureCollection = await this.sourceFeatureType.getFeaturesByCursorAndLimit(cursor, this.fetchSize);
        let targetFeatureCollection;

        featureCount = sourceFeatureCollection.features.length;
        if(featureCount == 0){
          console.log('No more cells.');
          break;
        }
        // @ts-ignore
        cursor = parseInt(sourceFeatureCollection.features[sourceFeatureCollection.features.length-1].properties.id);

        console.time('computeArea');
        let idAreaMap: Map<any,number>;
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
          // @ts-ignore
          let idArea:[any,number][] = sourceFeatureCollection.features.map((feature: Feature)=>[feature.properties.id, turfArea(<Geometry>feature.geometry)]);
          idAreaMap = new Map(idArea);
        }
        console.timeEnd('computeArea');

        //build bpolys and add to ohsomeQuery
        //api requires ids to be strings
        //sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
        this.config.ohsomeQuery.bpolys = sourceFeatureCollection;


        console.time('query');
        let ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
        console.timeEnd('query');
        console.log('Start conversion ohsomeJSON to GeoJSON');
        console.time('convert');
        if (shouldCreateGeometry) {
            if (typeof ohsomeResults.data == 'string') {
              console.log('-----------------------------------------');
              console.log(ohsomeResults.data.substring(0,400));
              console.log('-----------------------------------------');
            }
            targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);

        } else {
            if (typeof ohsomeResults.data == 'string') {
              console.log('-----------------------------------------');
              console.log(ohsomeResults.data.substring(0,400));
              console.log('-----------------------------------------');
            }
            targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);

        }
        console.timeEnd('convert');

        if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues){
          //remove features where value = 0
         console.log('Remove Zeros');
          console.time('removeZero');
          targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection,'value', 0);
          console.timeEnd('removeZero');
        }

        console.time('computeValuePerArea');
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){

          targetFeatureCollection.features.forEach(
              // @ts-ignore
              (feature)=>feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
          );
        }
        console.timeEnd('computeValuePerArea');

        await this.targetFeatureType.writeFeatures(targetFeatureCollection);
        //await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);

      }
    } catch (e) {
      console.log(e);
      this.sourceFeatureType.finalize();
      this.targetFeatureType.finalize();
      throw new Error('Could not create ohsome data.')
    }

    } else {
      // create all at once
      try {
        let targetFeatureCollection;
        const sourceFeatureCollection = await this.sourceFeatureType.getFeatures();

        console.time('computeArea');
        let idAreaMap: Map<any,number>;
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){
          let idArea = sourceFeatureCollection.features.map((feature:any)=>[feature.properties.id, turfArea(feature.geometry)]);
          idAreaMap = new Map(idArea);
        }
        console.timeEnd('computeArea');

        //build bpolys and add to ohsomeQuery
        //api requires ids to be strings
        //sourceFeatureCollection.features.forEach((feature)=>{feature.properties.id = String(feature.properties.id)});
        this.config.ohsomeQuery.bpolys = sourceFeatureCollection;

        console.time('query');
        const  ohsomeResults = await this.getOhsomeResults(this.config.ohsomeQuery);
        console.timeEnd('query');

        console.log('Start conversion ohsomeJSON or csv to GeoJSON');
        console.time('convert');
        if (shouldCreateGeometry) {
          if (typeof ohsomeResults.data == 'string') {
            console.log('-----------------------------------------');
            console.log(ohsomeResults.data.substring(0,400));
            console.log('-----------------------------------------');
          }
            targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps, sourceFeatureCollection, transformToWebmercator);

        } else {

          if (typeof ohsomeResults.data == 'string') {
            console.log('-----------------------------------------');
            console.log(ohsomeResults.data.substring(0,400));
            console.log('-----------------------------------------');
          }
          targetFeatureCollection = GeoJsonFeatureType.fromOhsome(ohsomeResults.data, shouldWriteHorizontalTimestamps);

        }
        console.timeEnd('convert');

        if (!shouldWriteHorizontalTimestamps && !this.storeZeroValues){
          //remove features where value = 0

          targetFeatureCollection = GeoJsonFeatureType.removeFeaturesByPropertyValue(targetFeatureCollection,'value', 0);
        }

        console.time('computeValuePerArea');
        if(!shouldWriteHorizontalTimestamps && this.computeValuePerArea){

          targetFeatureCollection.features.forEach(
              // @ts-ignore
              (feature)=>feature.properties["value_per_area"] = feature.properties.value / idAreaMap.get(feature.properties.id)
          );
        }
        console.timeEnd('computeValuePerArea');

        await this.targetFeatureType.writeFeatures(targetFeatureCollection);

        //await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);
      } catch (e) {
        console.log(e);
        this.sourceFeatureType.finalize();
        this.targetFeatureType.finalize();
        throw new Error('Could not create ohsome data.');
      }

    }

    //createIndexes?
    try {
      // if(this.targetFeatureType.store.type == 'postgis' && !!this.config.target.createIndexes){
        if(this.targetFeatureType instanceof PgFeatureType) {
          this.targetFeatureType = <PgFeatureType>this.targetFeatureType;
          let pgTarget = <PostgisFeatureTypeConfig>this.config.target;
          if( !! pgTarget.createIndexes){
        await this.targetFeatureType.createIndex('id');
        if(!!this.config.target.horizontalTimestampColumns){
          // TODO:horizontal
          console.log('create indexes for horizontalTimestampColumns not yet implemented!');
        } else {
          // vertical: one timestamp column
          await this.targetFeatureType.createIndex('timestamp');

          await this.targetFeatureType.clusterTable('timestamp');

          await this.targetFeatureType.createIndex('value');

          if (this.computeValuePerArea) {
            await this.targetFeatureType.createIndex('value_per_area');
          }
        }

        if(!!this.config.target.createGeometry){
          await this.targetFeatureType.createIndex('geom', 'gist');
        }

        await this.targetFeatureType.analyzeTable();
      }}
    } catch (e) {
      console.log(e);
      this.sourceFeatureType.finalize();
      this.targetFeatureType.finalize();
      throw new Error('Could not create indexes.');
    }

    //finalize e.g. close connections
    this.sourceFeatureType.finalize();
    this.targetFeatureType.finalize();

    this.log_end = new Date();
    console.log('Finished at: ' + this.log_end.toLocaleString());
    let log_diff = this.log_end.valueOf() - this.log_start.valueOf();
    let diffDays = Math.floor(log_diff / 86400000); // days
    let diffHrs = Math.floor((log_diff % 86400000) / 3600000); // hours
    let diffMins = Math.round(((log_diff % 86400000) % 3600000) / 60000); // minutes
    let diffSecs = Math.round(((log_diff % 86400000) % 3600000) % 60000 / 1000 ); // seconds
    console.log('====================================================================================');
    console.log(`Duration: ${diffDays} days ${diffHrs} hours ${diffMins} min ${diffSecs} seconds` );
    console.log('====================================================================================');
  }

///////////// methods


  async getOhsomeResults(ohsomeQuery: OhsomeQueryConfig): Promise<AxiosResponse>{
//     axios.interceptors.request.use(request => {
//   console.log('Starting Request', request)
//   return request
// });
    const keys = ohsomeQuery.keys || '';
    const values = ohsomeQuery.values || '';
    const format = ohsomeQuery.format || 'json';
    try {
      console.log('----------------------------------');
      console.log('Stringify Cells');
      console.time('stringify');
      if(ohsomeQuery.bpolys == null){
        throw Error('bpolys undefined in OhsomeQueryConfig');
      }
      let bpolyString = JSON.stringify(ohsomeQuery.bpolys);
      console.timeEnd('stringify');
      console.log('----------------------------------');
      console.log('Stringify querystring');
      console.time('querystring');
      let dataString = querystring.stringify({
        bpolys: bpolyString,
        keys: keys,
        values: values,
        time: ohsomeQuery.time,
        types: ohsomeQuery.types,
        showMetadata: true,
        format: format
      });
      console.timeEnd('querystring');
      console.log('----------------------------------');
      console.log(`Querying ohsome-API: ${this.ohsomeApiUrl}/${ohsomeQuery.queryType}/groupBy/boundary`);
      const stats = await axios(<AxiosRequestConfig>{
          url: `${this.ohsomeApiUrl}/${ohsomeQuery.queryType}/groupBy/boundary`,
          method: 'post',
          header: {'content-type': 'application/x-www-form-urlencoded'},
          maxContentLength: 1024*1024*1024*1024,
          data: dataString
      });
      console.log('----------------------------------');
      console.log('Response Metadata', JSON.stringify(stats.data.metadata));
      console.log('----------------------------------');
      return stats;
    } catch (e) {
        console.log('Ohsome API request failed.');
        this.axiosErrorHandler(e);
        throw new Error(e.message);
    }

  }

  axiosErrorHandler(error:any){
    if (error.response) {
      // The request was made and the server responded with a status code
      // that falls out of the range of 2xx
        console.log('Response error status: ' + error.response.status);
        console.log('Response error headers: ' + JSON.stringify(error.response.headers,null,2));
        console.log('Response error data: ' + JSON.stringify(error.response.data,null,2));
    } else if (error.request) {
      // The request was made but no response was received
      // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
      // http.ClientRequest in node.js
      console.log('No response received, request.');
    } else {
      // Something happened in setting up the request that triggered an Error
      console.log('Error', error.message);
    }
    delete error.config.data; //avoid bpoly logging
    console.log('Request config:\n', error.config);
  }

   onError(error:any) {
    console.log(error);
    if (this.sourceFeatureType && this.targetFeatureType){
      this.sourceFeatureType.finalize();
      this.targetFeatureType.finalize();
    }
  }

}

export = Ohsome2X;