Skip to content

GitLab

Commit 22651ee5 authored by Michael Auer's avatar Michael Auer
Browse files

code cleanup

parent 3875f696
Hide whitespace changes
Inline Side-by-side
ohsome2x-cli/ohsome2x-cli.js
View file @ 22651ee5
......@@ -5,19 +5,6 @@ const fs = require('fs');
const Ohsome2X = require('../dist/Ohsome2X.js');
const argv = require('minimist')(process.argv.slice(2));
const inquirer = require('inquirer');
// const argv_ = process.argv.slice(2);
// console.dir(argv);
// console.log(argv_);
// node ohsome2x-cli.js run -s sourceConfig.json -t targetConfig.json -q ohsomeQuery.json
// node ohsome2x-cli.js run (-c|--conf) fullConfig.json
// node ohsome2x-cli.js run -i // IDEA: ask for configs
// node ohsome2x-cli.js createconfig [(-o|--out) path] // IDEA: interactively create and store featureTypeConfig
//console.log(process.cwd());
// path.basename(process.cwd());
// TODO: remove true for other subcommands
run().catch(console.log);
......
src/FeatureType.ts
View file @ 22651ee5
......@@ -3,10 +3,10 @@ import {FeatureCollection} from '@turf/helpers';
//abstract class and factory for postgis, GeoJSON, ...
export abstract class FeatureType {
private config: FeatureTypeConfig;
private config: FeatureTypeConfig<any>;
constructor(config: FeatureTypeConfig) {
protected constructor(config: FeatureTypeConfig<any>) {
this.config = config;
}
......@@ -17,8 +17,6 @@ export abstract class FeatureType {
abstract async writeFeatures(featureCollection: FeatureCollection): Promise<void>;
abstract async writeOhsomeFeatures(ohsomeGroupByBoundaryResponseJSON: any, horizontalTimestampColumns: boolean): Promise<void>;
/**
* physically deletes the feature type from the store (e.g. file from disk or table from database)
**/
......
src/FeatureTypeFactory.ts
View file @ 22651ee5
......@@ -9,7 +9,7 @@ import {FeatureTypeConfig, GeoJSONFeatureTypeConfig, PostgisFeatureTypeConfig} f
export class FeatureTypeFactory {
//factory
static async create(featureTypeConfig: FeatureTypeConfig){
static async create(featureTypeConfig: FeatureTypeConfig<any>){
const type = featureTypeConfig.store.type.toLowerCase();
switch (type) {
......@@ -32,31 +32,5 @@ export class FeatureTypeFactory {
}
}
static async checkTableExists(featureTypeConfig: PostgisFeatureTypeConfig){
const sql = `SELECT EXISTS (
SELECT 1
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = '${featureTypeConfig.schema || 'public'}'
AND c.relname = '${featureTypeConfig.name}'
AND c.relkind = 'r' -- only tables
);`;
let db = new PgAsync(featureTypeConfig.store);
let exists;
try {
exists = await db.value(sql);
console.log(`Table ${featureTypeConfig.schema}.${featureTypeConfig.name} exists: ${exists}`);
return exists;
} catch (e) {
console.log(e);
throw new Error('Could not check if table exists. Is the database accessible?');
} finally {
db.closeConnections();
}
}
}
src/GeoJsonFeatureType.ts
View file @ 22651ee5
......@@ -7,7 +7,11 @@ import * as turfInvariant from '@turf/invariant';
import * as Papa from 'papaparse';
import {reproject} from 'reproject';
import * as proj4 from 'proj4';
import {GeoJSONFeatureTypeConfig, GeoJSONStore} from './config_types_interfaces';
import {
GeoJSONStore,
SourceGeoJSONFeatureTypeConfig,
TargetGeoJSONFeatureTypeConfig
} from './config_types_interfaces';
//const fs = require('fs');
......@@ -22,18 +26,20 @@ export class GeoJsonFeatureType extends FeatureType {
public readonly store: GeoJSONStore;
private name: string;
private geometryId: string | number;
private geometryId: string | number | undefined;
private path: string;
private fileExists: boolean;
constructor(config: GeoJSONFeatureTypeConfig) {
constructor(config: SourceGeoJSONFeatureTypeConfig | TargetGeoJSONFeatureTypeConfig ) {
super(config);
this.store = config.store; //type:'geojson'
this.name = config.name || path.basename(config.store.path);
this.geometryId = config.geometryId || 'id';
if ('geometryId' in config) {
this.geometryId = config.geometryId || 'id';
}
this.path = config.store.path;
this.fileExists = fs.existsSync(this.path);
console.log('file exists: ' + this.fileExists);
console.log(`Checked if FILE ${this.path} exists: ${this.fileExists}`);
}
async delete() {
......@@ -71,13 +77,16 @@ export class GeoJsonFeatureType extends FeatureType {
}
//normalize id columnName
if (this.geometryId != 'id') {
if (typeof this.geometryId != 'undefined' && this.geometryId != 'id') {
geojson.features.forEach(
(feature: Feature) => {
// @ts-ignore
if (feature.properties === null || !feature.properties[this.geometryId]) {
throw new Error(`No id column "${this.geometryId}" found on ${this.name}. Please configure geometryId: '<columnNameOfId>' in featureTypeConfig.`);
}
// @ts-ignore
feature.properties.id = feature.properties[this.geometryId];
// @ts-ignore
delete feature.properties[this.geometryId]
}, this);
}
......@@ -328,10 +337,6 @@ export class GeoJsonFeatureType extends FeatureType {
return targetFeatureCollection;
}
writeOhsomeFeatures(ohsomeGroupByBoundaryResponseJSON: any, horizontalTimestampColumns: boolean): Promise<void> {
throw new Error("Method not implemented.");
}
finalize(): Promise<void> {
return Promise.resolve();
}
......
src/PgFeatureType.ts
View file @ 22651ee5
......@@ -7,7 +7,11 @@
// const moment = require('moment');
//extends FeatureType
import {PostgisFeatureTypeConfig, PostgisStore} from './config_types_interfaces';
import {
PostgisStore,
SourcePostgisFeatureTypeConfig,
TargetPostgisFeatureTypeConfig
} from './config_types_interfaces';
import {FeatureType} from './FeatureType';
import PgAsync from 'pg-async';
import {types as pgTypes} from 'pg';
......@@ -26,21 +30,25 @@ export class PgFeatureType extends FeatureType {
public readonly store: PostgisStore;
private schemaName: string;
private tableName: string;
private geometryId: string | number;
private geometryId: string | number | undefined;
private geometryColumn: string;
private fetchSize: number | undefined;
private db: PgAsync;
private tableExists: any;
constructor(config: PostgisFeatureTypeConfig) {
constructor(config: SourcePostgisFeatureTypeConfig | TargetPostgisFeatureTypeConfig) {
super(config);
// this.config = config;
this.store = config.store; //type:'postgis'
this.schemaName = config.schema || 'public';
this.tableName = config.name;
this.geometryId = config.geometryId;
if ('geometryId' in config) {
this.geometryId = config.geometryId;
}
this.geometryColumn = config.geometryColumn;
this.fetchSize = config.fetchSize;
if ('fetchSize' in config) {
this.fetchSize = config.fetchSize;
}
//per default pg parses int8 values as strings
pgTypes.setTypeParser(20, function (val) {
return parseInt(val)
......@@ -69,7 +77,7 @@ export class PgFeatureType extends FeatureType {
let exists;
try {
exists = await this.db.value(sql);
console.log(`Table ${this.schemaName}.${this.tableName} exists: ${exists}`);
console.log(`Checked if TABLE ${this.schemaName}.${this.tableName} exists: ${exists}`);
return exists;
} catch (e) {
console.log(e);
......@@ -167,36 +175,6 @@ export class PgFeatureType extends FeatureType {
return result;
}
async createTableFromOhsome(ohsomeGroupByBoundaryResponseJSON: any, horizontalTimestampColumns: boolean | null) {
horizontalTimestampColumns = !!horizontalTimestampColumns;
// will need this for horizontal timestamp columns let data = ohsomeGroupByBoundaryResponseJSON;
let sql: string;
if (horizontalTimestampColumns) {
// TODO:
throw Error('horizontalTimestampColumns: true not yet implemented in PgFeatureType.createTableFromOhsome() method');
} else {
//vertical: single timestamp column
// TODO: infer id type from input id; infer value type from count|length|...
let columns = 'id integer, "timestamp" timestamptz, value double precision';
sql = `CREATE TABLE IF NOT EXISTS ${this.schemaName}."${this.tableName}" (${columns});`;
}
let res;
try {
res = await this.db.query(sql);
this.tableExists = true;
console.log(sql);
} catch (e) {
console.log(e);
} finally {
return res;
}
}
async createTableFromGeoJSONFeature(geoJSONFeature: Feature, epsg?: number | string) {
//check if geometries exists
const hasGeometry = !!geoJSONFeature.geometry;
......@@ -284,7 +262,7 @@ export class PgFeatureType extends FeatureType {
if (hasGeometry) {
properties.push('geom')
}
;
const columnNames = properties.map((property) => `"${property}"`).join(',');
function castToDataType(value: any, index: number) {
......@@ -321,7 +299,7 @@ export class PgFeatureType extends FeatureType {
featureValues.push("ST_SetSRID(ST_GeomFromGeoJSON('" + JSON.stringify(feature.geometry) + "')," + epsg + ")");
}
;
return `(${featureValues.join(',')})`
}).join(',');
......@@ -333,54 +311,6 @@ export class PgFeatureType extends FeatureType {
}
async writeOhsomeFeatures(ohsomeGroupByBoundaryResponseJSON: any, horizontalTimestampColumns: boolean) {
//sets undefined to false, so default is vertical
horizontalTimestampColumns = !!horizontalTimestampColumns;
let data = ohsomeGroupByBoundaryResponseJSON;
let insertIntoSQL;
if (horizontalTimestampColumns) {
// horizontal: one column for each timestamp
if (!this.tableExists) {
await this.createTableFromOhsome(ohsomeGroupByBoundaryResponseJSON, true);
}
let columnNames = 'id,' + data.groupByResult[0].result.map((tv: any) => '"' + tv.timestamp + '"').join();
let values = data.groupByResult.map((item: any) => {
let id = [item.groupByObject];
let values = item.result.map((tv: any) => tv.value);
return "(" + id.concat(values).join() + ")";
}).join();
insertIntoSQL = `INSERT INTO ${this.schemaName}."${this.tableName}" (${columnNames} ) VALUES ${values};`;
} else {
//vertical: single timestamp column
if (!this.tableExists) {
await this.createTableFromOhsome(ohsomeGroupByBoundaryResponseJSON, false);
}
let columnNames = 'id, "timestamp", value';
let values = data.groupByResult.map((item: any) => {
let id = [item.groupByObject];
let values = item.result.map((tv: any) => {
return `(${id},'${tv.timestamp}'::timestamptz,${tv.value})`
});
return values
}).join();
insertIntoSQL = `INSERT INTO ${this.schemaName}."${this.tableName}" (${columnNames} ) VALUES ${values};`;
// console.log(insertIntoSQL);
}
console.log('Write ohsome features.');
return await this.db.query(insertIntoSQL);
}
async createIndex(column: string, indexType?: string) {
column = column.replace(' ', '_');
const usingIndexType = (!!indexType) ? `USING ${indexType}` : '';
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment