Verified Commit ef3d98c8 authored by Michael Auer's avatar Michael Auer
Browse files

closes #10 partitioning options added

parent 94d3d0d9
import {FeatureTypeConfig} from './config_types_interfaces';
import {
SourceFeatureTypeConfig,
TargetFeatureTypeConfig
} from './config_types_interfaces';
import {FeatureCollection} from '@turf/helpers';
import {PgColumnTypes} from './PgColumnTypes';
//abstract class and factory for postgis, GeoJSON, ...
export abstract class FeatureType {
protected config: FeatureTypeConfig<any>;
protected config: SourceFeatureTypeConfig | TargetFeatureTypeConfig;
protected keepColumns: { name:string; as?:string }[];
protected constructor(config: FeatureTypeConfig<any>) {
protected constructor(config: SourceFeatureTypeConfig | TargetFeatureTypeConfig) {
this.config = config;
if ('keepColumns' in config) {
//sourceConfig
......
......@@ -69,6 +69,8 @@ class Ohsome2X extends EventEmitter {
// write timestamp values in one or many columns? default is vertical (one column or two columns for contribution view queries)
this.shouldWriteHorizontalTimestamps = !!this.config.target.horizontalTimestampColumns;
this.ohsomeApiUrl = config.ohsomeApiUrl || OHSOME_API_URL;
//default use csv because its way faster than parsing json
......@@ -159,7 +161,6 @@ class Ohsome2X extends EventEmitter {
}
//dropTableIfexisits and views, do not delete if completing table from explicitly specified cursor
// @ts-ignore
if (this.targetFeatureType.store.type == 'postgis' && this.cursor == null) {
try {
await this.targetFeatureType.delete();
......@@ -335,7 +336,6 @@ class Ohsome2X extends EventEmitter {
await this.targetFeatureType.writeFeatures(targetFeatureCollection, targetPgColumnTypes);
//update cursor for the next round of fetches
// @ts-ignore
cursor = sourceFeatureCollection.features[sourceFeatureCollection.features.length - 1].properties.id;
//update finished featureCount
this.currentFeatureCount += featureCount;
......@@ -534,8 +534,8 @@ class Ohsome2X extends EventEmitter {
sourceFeatureCollection = <FeatureCollection>reproject(<FeatureCollection>sourceFeatureCollection, proj4.WGS84, "EPSG:3857", proj4.defs);
}
// @ts-ignore
sourceFeatureCollection.features.forEach((feature) => idGeomMap.set(feature.properties.id, feature.geometry));
sourceFeatureCollection.features.forEach((feature) => idGeomMap.set(feature.properties!.id, feature.geometry));
if (this.transformToWebmercator) {
// @ts-ignore
......
......@@ -25,6 +25,9 @@ export class PgFeatureType extends FeatureType {
public readonly store: PostgisStore;
private schemaName: string;
private tableName: string;
private shouldCreatePartitions: boolean = false;
private partitionBaseName: string;
private partitionBy: { key: string; values: any[]; createDefaultPartition: boolean; baseName?: string } | undefined;
private geometryId: string | number | undefined;
private geometryColumn: string;
......@@ -34,7 +37,7 @@ export class PgFeatureType extends FeatureType {
constructor(config: SourcePostgisFeatureTypeConfig | TargetPostgisFeatureTypeConfig) {
super(config);
// this.config = config;
this.store = config.store; //type:'postgis'
this.schemaName = config.schema || 'public';
this.tableName = config.name;
......@@ -43,6 +46,16 @@ export class PgFeatureType extends FeatureType {
}
this.geometryColumn = config.geometryColumn;
this.partitionBaseName = ('partitionBy' in config && config.partitionBy!.baseName != null)? config.partitionBy!.baseName : config.name;
// create List Partitions in case of target being a postgres DB
if ("partitionBy" in config) {
this.shouldCreatePartitions =
config.partitionBy!.key != null
&& config.partitionBy!.values.length > 0;
this.partitionBy = (this.shouldCreatePartitions)? config.partitionBy : undefined;
}
this.pgp = pgPromise({});
//per default pg parses int8 values as strings
......@@ -342,10 +355,31 @@ export class PgFeatureType extends FeatureType {
const columnDefsString = columnDefs.join(',');
const sql = `CREATE TABLE IF NOT EXISTS ${this.schemaName}."${this.tableName}" (${columnDefsString});`;
let partitionBySQL = '';
let partitionsCreateSQL = '';
if(this.shouldCreatePartitions){
partitionBySQL = `PARTITION BY LIST(${this.partitionBy!.key});`
partitionsCreateSQL = this.partitionBy!.values.map(
val => `CREATE TABLE IF NOT EXISTS ${this.schemaName}."${this.partitionBaseName}_${val}" PARTITION OF ${this.schemaName}."${this.tableName}" FOR VALUES IN (${val});`)
.join(`
`);
if(!!this.partitionBy!.createDefaultPartition){
partitionsCreateSQL += `
CREATE TABLE IF NOT EXISTS ${this.schemaName}."${this.partitionBaseName}_default"
PARTITION OF ${this.schemaName}."${this.tableName}" DEFAULT;
`
}
}
const sql = `CREATE TABLE IF NOT EXISTS ${this.schemaName}."${this.tableName}" (${columnDefsString})
${partitionBySQL}
${partitionsCreateSQL}
`;
console.log((sql.length > 70) ? sql.substring(0, 67) + '...' : sql);
// console.log(sql);
console.log(sql);
let res = await this.db.none(sql);
this.tableExists = true;
console.log(`✔️ TABLE is ready.`)
......@@ -354,7 +388,7 @@ export class PgFeatureType extends FeatureType {
/**
* @param featureCollection may or may not have geometries (geometry: null)
* @param pgTypeMappings
* @param pgColumnTypes
**/
//async writeFeatures(featureCollection: FeatureCollection, pgTypeMappings: object) {}
async writeFeatures(featureCollection: FeatureCollection, pgColumnTypes: PgColumnTypes): Promise<void> {
......
......@@ -31,7 +31,7 @@ export interface SourceFeatureTypeConfig {
//not reserved column names as id, timestamp, value, value2, ratio etc.
// input column names can be renamed using `{name: 'origname', as: 'newname'}` syntax
// e.g. [{name: 'res', as: 'resolution'}]
keepColumns?: { name:string; as?:string }[];
keepColumns?: { name: string; as?: string }[];
}
export interface TargetFeatureTypeConfig {
......@@ -61,6 +61,7 @@ export interface SourcePostgisFeatureTypeConfig extends PostgisFeatureTypeConfig
export interface TargetGeoJSONFeatureTypeConfig extends GeoJSONFeatureTypeConfig, TargetFeatureTypeConfig {}
export interface TargetPostgisFeatureTypeConfig extends PostgisFeatureTypeConfig, TargetFeatureTypeConfig {
createIndexes?: boolean;
partitionBy?: {key: string; values: any[], createDefaultPartition: boolean, baseName?: string}
}
export type OhsomeQueryConfigFormat = 'json' | 'csv' | undefined;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment