Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
giscience
big-data
ohsome
libs
ohsome2X
Commits
22651ee5
Commit
22651ee5
authored
May 03, 2020
by
Michael Auer
Browse files
code cleanup
parent
3875f696
Changes
5
Hide whitespace changes
Inline
Side-by-side
ohsome2x-cli/ohsome2x-cli.js
View file @
22651ee5
...
...
@@ -5,19 +5,6 @@ const fs = require('fs');
const
Ohsome2X
=
require
(
'
../dist/Ohsome2X.js
'
);
const
argv
=
require
(
'
minimist
'
)(
process
.
argv
.
slice
(
2
));
const
inquirer
=
require
(
'
inquirer
'
);
// const argv_ = process.argv.slice(2);
// console.dir(argv);
// console.log(argv_);
// node ohsome2x-cli.js run -s sourceConfig.json -t targetConfig.json -q ohsomeQuery.json
// node ohsome2x-cli.js run (-c|--conf) fullConfig.json
// node ohsome2x-cli.js run -i // IDEA: ask for configs
// node ohsome2x-cli.js createconfig [(-o|--out) path] // IDEA: interactively create and store featureTypeConfig
//console.log(process.cwd());
// path.basename(process.cwd());
// TODO: remove true for other subcommands
run
().
catch
(
console
.
log
);
...
...
src/FeatureType.ts
View file @
22651ee5
...
...
@@ -3,10 +3,10 @@ import {FeatureCollection} from '@turf/helpers';
//abstract class and factory for postgis, GeoJSON, ...
export
abstract
class
FeatureType
{
private
config
:
FeatureTypeConfig
;
private
config
:
FeatureTypeConfig
<
any
>
;
constructor
(
config
:
FeatureTypeConfig
)
{
protected
constructor
(
config
:
FeatureTypeConfig
<
any
>
)
{
this
.
config
=
config
;
}
...
...
@@ -17,8 +17,6 @@ export abstract class FeatureType {
abstract
async
writeFeatures
(
featureCollection
:
FeatureCollection
):
Promise
<
void
>
;
abstract
async
writeOhsomeFeatures
(
ohsomeGroupByBoundaryResponseJSON
:
any
,
horizontalTimestampColumns
:
boolean
):
Promise
<
void
>
;
/**
* physically deletes the feature type from the store (e.g. file from disk or table from database)
**/
...
...
src/FeatureTypeFactory.ts
View file @
22651ee5
...
...
@@ -9,7 +9,7 @@ import {FeatureTypeConfig, GeoJSONFeatureTypeConfig, PostgisFeatureTypeConfig} f
export
class
FeatureTypeFactory
{
//factory
static
async
create
(
featureTypeConfig
:
FeatureTypeConfig
){
static
async
create
(
featureTypeConfig
:
FeatureTypeConfig
<
any
>
){
const
type
=
featureTypeConfig
.
store
.
type
.
toLowerCase
();
switch
(
type
)
{
...
...
@@ -32,31 +32,5 @@ export class FeatureTypeFactory {
}
}
static
async
checkTableExists
(
featureTypeConfig
:
PostgisFeatureTypeConfig
){
const
sql
=
`SELECT EXISTS (
SELECT 1
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = '
${
featureTypeConfig
.
schema
||
'
public
'
}
'
AND c.relname = '
${
featureTypeConfig
.
name
}
'
AND c.relkind = 'r' -- only tables
);`
;
let
db
=
new
PgAsync
(
featureTypeConfig
.
store
);
let
exists
;
try
{
exists
=
await
db
.
value
(
sql
);
console
.
log
(
`Table
${
featureTypeConfig
.
schema
}
.
${
featureTypeConfig
.
name
}
exists:
${
exists
}
`
);
return
exists
;
}
catch
(
e
)
{
console
.
log
(
e
);
throw
new
Error
(
'
Could not check if table exists. Is the database accessible?
'
);
}
finally
{
db
.
closeConnections
();
}
}
}
src/GeoJsonFeatureType.ts
View file @
22651ee5
...
...
@@ -7,7 +7,11 @@ import * as turfInvariant from '@turf/invariant';
import
*
as
Papa
from
'
papaparse
'
;
import
{
reproject
}
from
'
reproject
'
;
import
*
as
proj4
from
'
proj4
'
;
import
{
GeoJSONFeatureTypeConfig
,
GeoJSONStore
}
from
'
./config_types_interfaces
'
;
import
{
GeoJSONStore
,
SourceGeoJSONFeatureTypeConfig
,
TargetGeoJSONFeatureTypeConfig
}
from
'
./config_types_interfaces
'
;
//const fs = require('fs');
...
...
@@ -22,18 +26,20 @@ export class GeoJsonFeatureType extends FeatureType {
public
readonly
store
:
GeoJSONStore
;
private
name
:
string
;
private
geometryId
:
string
|
number
;
private
geometryId
:
string
|
number
|
undefined
;
private
path
:
string
;
private
fileExists
:
boolean
;
constructor
(
config
:
GeoJSONFeatureTypeConfig
)
{
constructor
(
config
:
Source
GeoJSONFeatureTypeConfig
|
TargetGeoJSONFeatureTypeConfig
)
{
super
(
config
);
this
.
store
=
config
.
store
;
//type:'geojson'
this
.
name
=
config
.
name
||
path
.
basename
(
config
.
store
.
path
);
this
.
geometryId
=
config
.
geometryId
||
'
id
'
;
if
(
'
geometryId
'
in
config
)
{
this
.
geometryId
=
config
.
geometryId
||
'
id
'
;
}
this
.
path
=
config
.
store
.
path
;
this
.
fileExists
=
fs
.
existsSync
(
this
.
path
);
console
.
log
(
'
file
exists:
'
+
this
.
fileExists
);
console
.
log
(
`Checked if FILE
${
this
.
path
}
exists:
${
this
.
fileExists
}
`
);
}
async
delete
()
{
...
...
@@ -71,13 +77,16 @@ export class GeoJsonFeatureType extends FeatureType {
}
//normalize id columnName
if
(
this
.
geometryId
!=
'
id
'
)
{
if
(
typeof
this
.
geometryId
!=
'
undefined
'
&&
this
.
geometryId
!=
'
id
'
)
{
geojson
.
features
.
forEach
(
(
feature
:
Feature
)
=>
{
// @ts-ignore
if
(
feature
.
properties
===
null
||
!
feature
.
properties
[
this
.
geometryId
])
{
throw
new
Error
(
`No id column "
${
this
.
geometryId
}
" found on
${
this
.
name
}
. Please configure geometryId: '<columnNameOfId>' in featureTypeConfig.`
);
}
// @ts-ignore
feature
.
properties
.
id
=
feature
.
properties
[
this
.
geometryId
];
// @ts-ignore
delete
feature
.
properties
[
this
.
geometryId
]
},
this
);
}
...
...
@@ -328,10 +337,6 @@ export class GeoJsonFeatureType extends FeatureType {
return
targetFeatureCollection
;
}
writeOhsomeFeatures
(
ohsomeGroupByBoundaryResponseJSON
:
any
,
horizontalTimestampColumns
:
boolean
):
Promise
<
void
>
{
throw
new
Error
(
"
Method not implemented.
"
);
}
finalize
():
Promise
<
void
>
{
return
Promise
.
resolve
();
}
...
...
src/PgFeatureType.ts
View file @
22651ee5
...
...
@@ -7,7 +7,11 @@
// const moment = require('moment');
//extends FeatureType
import
{
PostgisFeatureTypeConfig
,
PostgisStore
}
from
'
./config_types_interfaces
'
;
import
{
PostgisStore
,
SourcePostgisFeatureTypeConfig
,
TargetPostgisFeatureTypeConfig
}
from
'
./config_types_interfaces
'
;
import
{
FeatureType
}
from
'
./FeatureType
'
;
import
PgAsync
from
'
pg-async
'
;
import
{
types
as
pgTypes
}
from
'
pg
'
;
...
...
@@ -26,21 +30,25 @@ export class PgFeatureType extends FeatureType {
public
readonly
store
:
PostgisStore
;
private
schemaName
:
string
;
private
tableName
:
string
;
private
geometryId
:
string
|
number
;
private
geometryId
:
string
|
number
|
undefined
;
private
geometryColumn
:
string
;
private
fetchSize
:
number
|
undefined
;
private
db
:
PgAsync
;
private
tableExists
:
any
;
constructor
(
config
:
PostgisFeatureTypeConfig
)
{
constructor
(
config
:
SourcePostgisFeatureTypeConfig
|
Target
PostgisFeatureTypeConfig
)
{
super
(
config
);
// this.config = config;
this
.
store
=
config
.
store
;
//type:'postgis'
this
.
schemaName
=
config
.
schema
||
'
public
'
;
this
.
tableName
=
config
.
name
;
this
.
geometryId
=
config
.
geometryId
;
if
(
'
geometryId
'
in
config
)
{
this
.
geometryId
=
config
.
geometryId
;
}
this
.
geometryColumn
=
config
.
geometryColumn
;
this
.
fetchSize
=
config
.
fetchSize
;
if
(
'
fetchSize
'
in
config
)
{
this
.
fetchSize
=
config
.
fetchSize
;
}
//per default pg parses int8 values as strings
pgTypes
.
setTypeParser
(
20
,
function
(
val
)
{
return
parseInt
(
val
)
...
...
@@ -69,7 +77,7 @@ export class PgFeatureType extends FeatureType {
let
exists
;
try
{
exists
=
await
this
.
db
.
value
(
sql
);
console
.
log
(
`
Table
${
this
.
schemaName
}
.
${
this
.
tableName
}
exists:
${
exists
}
`
);
console
.
log
(
`
Checked if TABLE
${
this
.
schemaName
}
.
${
this
.
tableName
}
exists:
${
exists
}
`
);
return
exists
;
}
catch
(
e
)
{
console
.
log
(
e
);
...
...
@@ -167,36 +175,6 @@ export class PgFeatureType extends FeatureType {
return
result
;
}
async
createTableFromOhsome
(
ohsomeGroupByBoundaryResponseJSON
:
any
,
horizontalTimestampColumns
:
boolean
|
null
)
{
horizontalTimestampColumns
=
!!
horizontalTimestampColumns
;
// will need this for horizontal timestamp columns let data = ohsomeGroupByBoundaryResponseJSON;
let
sql
:
string
;
if
(
horizontalTimestampColumns
)
{
// TODO:
throw
Error
(
'
horizontalTimestampColumns: true not yet implemented in PgFeatureType.createTableFromOhsome() method
'
);
}
else
{
//vertical: single timestamp column
// TODO: infer id type from input id; infer value type from count|length|...
let
columns
=
'
id integer, "timestamp" timestamptz, value double precision
'
;
sql
=
`CREATE TABLE IF NOT EXISTS
${
this
.
schemaName
}
."
${
this
.
tableName
}
" (
${
columns
}
);`
;
}
let
res
;
try
{
res
=
await
this
.
db
.
query
(
sql
);
this
.
tableExists
=
true
;
console
.
log
(
sql
);
}
catch
(
e
)
{
console
.
log
(
e
);
}
finally
{
return
res
;
}
}
async
createTableFromGeoJSONFeature
(
geoJSONFeature
:
Feature
,
epsg
?:
number
|
string
)
{
//check if geometries exists
const
hasGeometry
=
!!
geoJSONFeature
.
geometry
;
...
...
@@ -284,7 +262,7 @@ export class PgFeatureType extends FeatureType {
if
(
hasGeometry
)
{
properties
.
push
(
'
geom
'
)
}
;
const
columnNames
=
properties
.
map
((
property
)
=>
`"
${
property
}
"`
).
join
(
'
,
'
);
function
castToDataType
(
value
:
any
,
index
:
number
)
{
...
...
@@ -321,7 +299,7 @@ export class PgFeatureType extends FeatureType {
featureValues
.
push
(
"
ST_SetSRID(ST_GeomFromGeoJSON('
"
+
JSON
.
stringify
(
feature
.
geometry
)
+
"
'),
"
+
epsg
+
"
)
"
);
}
;
return
`(
${
featureValues
.
join
(
'
,
'
)}
)`
}).
join
(
'
,
'
);
...
...
@@ -333,54 +311,6 @@ export class PgFeatureType extends FeatureType {
}
async
writeOhsomeFeatures
(
ohsomeGroupByBoundaryResponseJSON
:
any
,
horizontalTimestampColumns
:
boolean
)
{
//sets undefined to false, so default is vertical
horizontalTimestampColumns
=
!!
horizontalTimestampColumns
;
let
data
=
ohsomeGroupByBoundaryResponseJSON
;
let
insertIntoSQL
;
if
(
horizontalTimestampColumns
)
{
// horizontal: one column for each timestamp
if
(
!
this
.
tableExists
)
{
await
this
.
createTableFromOhsome
(
ohsomeGroupByBoundaryResponseJSON
,
true
);
}
let
columnNames
=
'
id,
'
+
data
.
groupByResult
[
0
].
result
.
map
((
tv
:
any
)
=>
'
"
'
+
tv
.
timestamp
+
'
"
'
).
join
();
let
values
=
data
.
groupByResult
.
map
((
item
:
any
)
=>
{
let
id
=
[
item
.
groupByObject
];
let
values
=
item
.
result
.
map
((
tv
:
any
)
=>
tv
.
value
);
return
"
(
"
+
id
.
concat
(
values
).
join
()
+
"
)
"
;
}).
join
();
insertIntoSQL
=
`INSERT INTO
${
this
.
schemaName
}
."
${
this
.
tableName
}
" (
${
columnNames
}
) VALUES
${
values
}
;`
;
}
else
{
//vertical: single timestamp column
if
(
!
this
.
tableExists
)
{
await
this
.
createTableFromOhsome
(
ohsomeGroupByBoundaryResponseJSON
,
false
);
}
let
columnNames
=
'
id, "timestamp", value
'
;
let
values
=
data
.
groupByResult
.
map
((
item
:
any
)
=>
{
let
id
=
[
item
.
groupByObject
];
let
values
=
item
.
result
.
map
((
tv
:
any
)
=>
{
return
`(
${
id
}
,'
${
tv
.
timestamp
}
'::timestamptz,
${
tv
.
value
}
)`
});
return
values
}).
join
();
insertIntoSQL
=
`INSERT INTO
${
this
.
schemaName
}
."
${
this
.
tableName
}
" (
${
columnNames
}
) VALUES
${
values
}
;`
;
// console.log(insertIntoSQL);
}
console
.
log
(
'
Write ohsome features.
'
);
return
await
this
.
db
.
query
(
insertIntoSQL
);
}
async
createIndex
(
column
:
string
,
indexType
?:
string
)
{
column
=
column
.
replace
(
'
'
,
'
_
'
);
const
usingIndexType
=
(
!!
indexType
)
?
`USING
${
indexType
}
`
:
''
;
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment