Commit b720e13d authored by Michael Auer's avatar Michael Auer
Browse files

adding events for error, fetch, progress and finished

parent 7fca85ea
......@@ -11,7 +11,12 @@ export abstract class FeatureType {
}
/**
* @returns GeoJSON<FeatureCollection>
* @return number the total number of features
*/
abstract async getFeatureCount(): Promise<number>
/**
* @return GeoJSON<FeatureCollection>
**/
abstract async getFeatures(): Promise<FeatureCollection>;
......
......@@ -54,10 +54,18 @@ export class GeoJsonFeatureType extends FeatureType {
}
};
/**
* returns the total number of features
*/
async getFeatureCount(): Promise<number> {
const geojson = await this.getFeatures();
return geojson.features.length;
}
/**
* @returns GeoJSON<FeatureCollection>
**/
getFeatures() {
async getFeatures(): Promise<FeatureCollection> {
if (!fs.existsSync(this.path)) {
throw new Error('Cannot get features. File does not exist: ' + this.path);
}
......
......@@ -13,10 +13,11 @@ import * as querystring from 'querystring';
import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
import defaultConfig from './conf/default';
import {EventEmitter} from 'events';
let OHSOME_API_URL = normalizeUrl(defaultConfig.OHSOME_API_URL); //remove trailing slash and other things
class Ohsome2X {
class Ohsome2X extends EventEmitter {
private config: Ohsome2XConfig;
private cursor: number | string | undefined;
private fetchSize: number | null;
......@@ -28,8 +29,11 @@ class Ohsome2X {
private log_start: Date;
private log_end: Date;
private isContributionView: boolean | undefined;
private totalFeatureCount: number = 0;
private currentFeatureCount: number = 0;
constructor(config: Ohsome2XConfig) {
super(); //EventEmitter
this.config = config;
if ('cursor' in this.config.source) {
......@@ -67,12 +71,17 @@ class Ohsome2X {
async run() {
console.log('RUN');
//initialize featureTypes with info if table aleady exists;
try {
this.sourceFeatureType = await FeatureTypeFactory.create(this.config.source);
this.targetFeatureType = await FeatureTypeFactory.create(this.config.target);
this.totalFeatureCount = await this.sourceFeatureType.getFeatureCount();
this.emit('progress', {type:'progress', processed: this.currentFeatureCount, total: this.totalFeatureCount, timestamp: new Date().toISOString()});
} catch (e) {
console.log(e);
this.emit('error', {type:'error', error: e, message: 'Could not initialize FeatureTypes.', config: this.config, timestamp: new Date().toISOString()});
throw new Error('Could not initialize FeatureTypes.');
}
......@@ -87,7 +96,14 @@ class Ohsome2X {
//dropTableIfexisits and views, do not delete if completing table from explicitly specified cursor
// @ts-ignore
if (this.targetFeatureType.store.type == 'postgis' && this.cursor == null) {
await this.targetFeatureType.delete();
try {
await this.targetFeatureType.delete();
} catch (e) {
console.log(e);
this.emit('error', {type:'error', error: e, message: 'Could not delete existing PostGIS table.', config: this.config, timestamp: new Date().toISOString()});
throw new Error('Could not delete existing PostGIS table.');
}
}
//createItertively or createAllAtOnce (posgis source and tagrget only)
......@@ -120,10 +136,14 @@ class Ohsome2X {
try {
while (true) {
//emit event before fetching data
this.emit('fetch', {type: 'fetch', originalConfig: this.config, currentCursor: cursor, timestamp: new Date().toISOString()});
const sourceFeatureCollection: any = await this.sourceFeatureType.getFeaturesByCursorAndLimit(cursor, this.fetchSize);
let targetFeatureCollection;
featureCount = sourceFeatureCollection.features.length;
this.currentFeatureCount += featureCount;
if (featureCount == 0) {
console.log('No more cells.');
break;
......@@ -195,9 +215,11 @@ class Ohsome2X {
await this.targetFeatureType.writeFeatures(targetFeatureCollection);
console.log('-----------------------------------------' + Date.now() + '-----------------------------------------');
this.emit('progress', {type:'progress', processed: this.currentFeatureCount, total: this.totalFeatureCount, timestamp: new Date().toISOString()});
}
} catch (e) {
console.log(e);
this.emit('error', {type:'error', error: e, message: 'Could not create ohsome data.', config: this.config, timestamp: new Date().toISOString()});
this.sourceFeatureType.finalize();
this.targetFeatureType.finalize();
throw new Error('Could not create ohsome data.')
......@@ -209,11 +231,13 @@ class Ohsome2X {
let targetFeatureCollection;
const sourceFeatureCollection = await this.sourceFeatureType.getFeatures();
this.currentFeatureCount = sourceFeatureCollection.features.length;
console.time('computeArea');
let idAreaMap: Map<any, number>;
//in horizontal timestamp layout we can't have value and value_per_area, only one value is possible
if (!shouldWriteHorizontalTimestamps && this.computeValuePerArea) {
let idArea = sourceFeatureCollection.features.map((feature: any) => [feature.properties.id, turfArea(feature.geometry)]);
let idArea: any[any][number] = sourceFeatureCollection.features.map((feature: any) => [feature.properties.id, turfArea(feature.geometry)]);
idAreaMap = new Map(idArea);
}
console.timeEnd('computeArea');
......@@ -269,9 +293,10 @@ class Ohsome2X {
// console.log(JSON.stringify(targetFeatureCollection, undefined, 2));
await this.targetFeatureType.writeFeatures(targetFeatureCollection);
//await this.targetFeatureType.writeOhsomeFeatures(ohsomeResults.data, false);
this.emit('progress', {type:'progress', processed: this.currentFeatureCount, total: this.totalFeatureCount, timestamp: new Date().toISOString()});
} catch (e) {
console.log(e);
this.emit('error', {type:'error', error: e, message: 'Could not create ohsome data.', config: this.config, timestamp: new Date().toISOString()});
this.sourceFeatureType.finalize();
this.targetFeatureType.finalize();
throw new Error('Could not create ohsome data.');
......@@ -279,6 +304,8 @@ class Ohsome2X {
}
//createIndexes?
try {
// if(this.targetFeatureType.store.type == 'postgis' && !!this.config.target.createIndexes){
......@@ -323,6 +350,7 @@ class Ohsome2X {
}
} catch (e) {
console.log(e);
this.emit('error', {type:'error', error: e, message: 'Could not create indexes.', config: this.config, timestamp: new Date().toISOString()});
this.sourceFeatureType.finalize();
this.targetFeatureType.finalize();
throw new Error('Could not create indexes.');
......@@ -342,6 +370,8 @@ class Ohsome2X {
console.log('====================================================================================');
console.log(`Duration: ${diffDays} days ${diffHrs} hours ${diffMins} min ${diffSecs} seconds`);
console.log('====================================================================================');
this.emit('finished', {type: 'finished', duration:`${diffDays} days ${diffHrs} hours ${diffMins} min ${diffSecs} seconds`, timestamp: this.log_end.toISOString()})
}
///////////// methods
......
......@@ -113,6 +113,15 @@ export class PgFeatureType extends FeatureType {
}
}
/**
* returns the total number of features
*/
async getFeatureCount(): Promise<number> {
const sql = `SELECT count(*)
FROM ${this.schemaName}."${this.tableName}";`;
return await this.db.value(sql);
}
/**
get all Features
id column always returned as string
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment