Skip to content

Commit

Permalink
feat: Add VectorSource (#3012)
Browse files Browse the repository at this point in the history
  • Loading branch information
ibgreen committed May 17, 2024
1 parent e4121df commit 490219e
Show file tree
Hide file tree
Showing 14 changed files with 1,418 additions and 150 deletions.
6 changes: 4 additions & 2 deletions modules/core/src/lib/api/create-data-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ import {DataSource, Source} from '@loaders.gl/loader-utils';
/**
* Creates a source from a service
* If type is not supplied, will try to automatically detect the the
* @param url URL to the image source
* @param url URL to the data source
* @param type type of source. if not known, set to 'auto'
* @returns an ImageSource instance
* @returns an DataSource instance
*/
export function createDataSource<
DataSourcePropsT extends DataSourceProps = DataSourceProps,
Expand All @@ -25,6 +25,8 @@ export function createDataSource<
return source.createDataSource(data, props) as DataSourceT;
}

// TODO - use selectSource...

/** Guess service type from URL */
function selectSource(url: string | Blob, sources: Source[]): Source | null {
for (const service of sources) {
Expand Down
40 changes: 35 additions & 5 deletions modules/flatgeobuf/src/flatgeobuf-loader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import type {GeoJSONTable, BinaryFeatureCollection} from '@loaders.gl/schema';
import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
import {parseFlatGeobuf, parseFlatGeobufInBatches} from './lib/parse-flatgeobuf';
import {
parseFlatGeobuf,
parseFlatGeobufInBatches,
ParseFlatGeobufOptions
} from './lib/parse-flatgeobuf';

// __VERSION__ is injected by babel-plugin-version-inline
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
Expand All @@ -17,13 +22,15 @@ export type FlatGeobufLoaderOptions = LoaderOptions & {
shape?: 'geojson-table' | 'columnar-table' | 'binary';
/** Override the URL to the worker bundle (by default loads from unpkg.com) */
workerUrl?: string;
boundingBox?: [[number, number], [number, number]];
};
gis?: {
reproject?: boolean;
_targetCrs?: string;
};
};

/** Load flatgeobuf on a worker */
export const FlatGeobufWorkerLoader = {
dataType: null as any,
batchType: null as any,
Expand All @@ -45,13 +52,36 @@ export const FlatGeobufWorkerLoader = {
reproject: false
}
}
} as const satisfies Loader<any, any, FlatGeobufLoaderOptions>;
} as const satisfies Loader<GeoJSONTable | BinaryFeatureCollection, any, FlatGeobufLoaderOptions>;

export const FlatGeobufLoader = {
...FlatGeobufWorkerLoader,
parse: async (arrayBuffer, options) => parseFlatGeobuf(arrayBuffer, options),
parseSync: parseFlatGeobuf,
parse: async (arrayBuffer: ArrayBuffer, options: FlatGeobufLoaderOptions = {}) =>
parseSync(arrayBuffer, options),
parseSync,
// @ts-expect-error this is a stream parser not an async iterator parser
parseInBatchesFromStream: parseFlatGeobufInBatches,
parseInBatchesFromStream,
binary: true
} as const satisfies LoaderWithParser<any, any, FlatGeobufLoaderOptions>;

function parseSync(arrayBuffer: ArrayBuffer, options: FlatGeobufLoaderOptions = {}) {
return parseFlatGeobuf(arrayBuffer, getOptions(options));
}

function parseInBatchesFromStream(stream: any, options: FlatGeobufLoaderOptions) {
return parseFlatGeobufInBatches(stream, getOptions(options));
}

function getOptions(options: FlatGeobufLoaderOptions): ParseFlatGeobufOptions {
options = {
...options,
flatgeobuf: {...FlatGeobufLoader.options.flatgeobuf, ...options?.flatgeobuf},
gis: {...FlatGeobufLoader.options.gis, ...options?.gis}
};
return {
shape: options?.flatgeobuf?.shape ?? 'geojson-table',
boundingBox: options?.flatgeobuf?.boundingBox,
crs: options?.gis?._targetCrs || 'WGS84',
reproject: options?.gis?.reproject || false
};
}
98 changes: 98 additions & 0 deletions modules/flatgeobuf/src/floatgeobuf-source.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors

import {Schema, GeoJSONTable} from '@loaders.gl/schema';
import type {
VectorSourceProps,
VectorSourceMetadata,
GetFeaturesParameters,
LoaderWithParser
} from '@loaders.gl/loader-utils';
import {Source, VectorSource} from '@loaders.gl/loader-utils';

import {FlatGeobufLoader} from './flatgeobuf-loader';

/**
* @ndeprecated This is a WIP, not fully implemented
* @see https://developers.arcgis.com/rest/services-reference/enterprise/feature-service.htm
*/
export const FlatGeobufSource = {
name: 'FlatGeobuf',
id: 'flatgeobuf-server',
module: 'wms',
version: '0.0.0',
extensions: [],
mimeTypes: [],
options: {
url: undefined!,
'flatgeobuf-server': {
/** Tabular loaders, normally the GeoJSONLoader */
loaders: []
}
},

type: 'flatgeobuf-server',

testURL: (url: string): boolean => url.toLowerCase().includes('FeatureServer'),
createDataSource: (url, props: FlatGeobufVectorSourceProps): FlatGeobufVectorSource =>
new FlatGeobufVectorSource(props)
} as const satisfies Source<FlatGeobufVectorSource, FlatGeobufVectorSourceProps>;

export type FlatGeobufVectorSourceProps = VectorSourceProps & {
url: string;
'flatgeobuf-server'?: {
loaders: LoaderWithParser[];
};
};

/**
* ArcGIS ImageServer
* Note - exports a big API, that could be exposed here if there is a use case
* @see https://developers.arcgis.com/rest/services-reference/enterprise/feature-service.htm
*/
export class FlatGeobufVectorSource extends VectorSource<FlatGeobufVectorSourceProps> {
data: string;
url: string;
protected formatSpecificMetadata: Promise<any> | null = null;

constructor(props: FlatGeobufVectorSourceProps) {
super(props);
this.data = props.url;
this.url = props.url;
// this.formatSpecificMetadata = this._getFormatSpecificMetadata();
}

/** TODO - not yet clear if we can find schema information in the FeatureServer metadata or if we need to request a feature */
async getSchema(): Promise<Schema> {
await this.getMetadata({formatSpecificMetadata: true});
return {metadata: {}, fields: []};
}

async getMetadata(options: {formatSpecificMetadata}): Promise<VectorSourceMetadata> {
// Wait for raw metadata to load
if (!this.formatSpecificMetadata) {
// this.formatSpecificMetadata = await this._getFormatSpecificMetadata();
}

// const metadata = parseFlatGeobufMetadata(this.formatSpecificMetadata);

// Only add the big blob of source metadata if explicitly requested
if (options.formatSpecificMetadata) {
// metadata.formatSpecificMetadata = this.formatSpecificMetadata;
}
// @ts-expect-error
return {};
}

async getFeatures(parameters: GetFeaturesParameters): Promise<GeoJSONTable> {
const response = await this.fetch(this.url);
const arrayBuffer = await response.arrayBuffer();
// TODO - hack - done to avoid pulling in selectLoader from core

const table = await FlatGeobufLoader.parse(arrayBuffer, {});
// const loader = this.props['flatgeobuf-server']?.loaders?.[0];
// const table = loader?.parse(arrayBuffer);
return table as GeoJSONTable;
}
}
101 changes: 61 additions & 40 deletions modules/flatgeobuf/src/lib/parse-flatgeobuf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import {Proj4Projection} from '@math.gl/proj4';
import {transformGeoJsonCoords} from '@loaders.gl/gis';

import type {FlatGeobufLoaderOptions} from '../flatgeobuf-loader';
import type {GeoJSONTable, Table, Schema} from '@loaders.gl/schema';

import {fgbToBinaryGeometry} from './binary-geometries';
Expand All @@ -20,34 +19,24 @@ const deserializeGeoJson = geojson.deserialize;
const deserializeGeneric = generic.deserialize;
// const parsePropertiesBinary = FlatgeobufFeature.parseProperties;

// TODO: reproject binary features
function binaryFromFeature(feature: fgb.Feature, header: fgb.HeaderMeta) {
const geometry = feature.geometry();

// FlatGeobuf files can only hold a single geometry type per file, otherwise
// GeometryType is GeometryCollection
// I believe geometry.type() is null (0) except when the geometry type isn't
// known in the header?
const geometryType = header.geometryType || geometry?.type();
const parsedGeometry = fgbToBinaryGeometry(geometry, geometryType!);
// @ts-expect-error this looks wrong
parsedGeometry.properties = parsePropertiesBinary(feature, header.columns);

// TODO: wrap binary data either in points, lines, or polygons key
return parsedGeometry;
}
export type ParseFlatGeobufOptions = {
shape?: 'geojson-table' | 'columnar-table' | 'binary';
/** If supplied, only loads features within the bounding box */
boundingBox?: [[number, number], [number, number]];
/** Desired output CRS */
crs?: string;
/** Should geometries be reprojected to target CRS */
reproject?: boolean;
};

/*
* Parse FlatGeobuf arrayBuffer and return GeoJSON.
*
* @param arrayBuffer A FlatGeobuf arrayBuffer
* @return A GeoJSON geometry object
*/
export function parseFlatGeobuf(
arrayBuffer: ArrayBuffer,
options?: FlatGeobufLoaderOptions
): Table {
const shape = options?.flatgeobuf?.shape;
export function parseFlatGeobuf(arrayBuffer: ArrayBuffer, options: ParseFlatGeobufOptions): Table {
const shape = options.shape;

switch (shape) {
case 'geojson-table': {
Expand All @@ -68,7 +57,7 @@ export function parseFlatGeobuf(
}
}

function parseFlatGeobufToBinary(arrayBuffer: ArrayBuffer, options: FlatGeobufLoaderOptions = {}) {
function parseFlatGeobufToBinary(arrayBuffer: ArrayBuffer, options: ParseFlatGeobufOptions = {}) {
// TODO: reproject binary features
// const {reproject = false, _targetCrs = 'WGS84'} = (options && options.gis) || {};

Expand All @@ -79,32 +68,34 @@ function parseFlatGeobufToBinary(arrayBuffer: ArrayBuffer, options: FlatGeobufLo

function parseFlatGeobufToGeoJSONTable(
arrayBuffer: ArrayBuffer,
options: FlatGeobufLoaderOptions = {}
options: ParseFlatGeobufOptions
): GeoJSONTable {
if (arrayBuffer.byteLength === 0) {
return {shape: 'geojson-table', type: 'FeatureCollection', features: []};
}

const {reproject = false, _targetCrs = 'WGS84'} = (options && options.gis) || {};
const {reproject = false, crs = 'WGS84'} = options;

const arr = new Uint8Array(arrayBuffer);

let fgbHeader;
let fgbHeader: fgb.HeaderMeta | undefined;
let schema: Schema | undefined;

const rect = options.boundingBox && convertBoundingBox(options.boundingBox);

// @ts-expect-error this looks wrong
let {features} = deserializeGeoJson(arr, undefined, (headerMeta) => {
let {features} = deserializeGeoJson(arr, rect, (headerMeta) => {
fgbHeader = headerMeta;
schema = getSchemaFromFGBHeader(fgbHeader);
});

const crs = fgbHeader && fgbHeader.crs;
const fromCRS = fgbHeader?.crs?.wkt;
let projection;
if (reproject && crs) {
if (reproject && fromCRS) {
// Constructing the projection may fail for some invalid WKT strings
try {
projection = new Proj4Projection({from: crs.wkt, to: _targetCrs});
} catch (e) {
projection = new Proj4Projection({from: fromCRS, to: crs});
} catch (error) {
// no op
}
}
Expand All @@ -123,8 +114,8 @@ function parseFlatGeobufToGeoJSONTable(
* @return A GeoJSON geometry object iterator
*/
// eslint-disable-next-line complexity
export function parseFlatGeobufInBatches(stream, options: FlatGeobufLoaderOptions) {
const shape = options.flatgeobuf?.shape;
export function parseFlatGeobufInBatches(stream, options: ParseFlatGeobufOptions) {
const shape = options.shape;
switch (shape) {
case 'binary':
return parseFlatGeobufInBatchesToBinary(stream, options);
Expand All @@ -135,12 +126,14 @@ export function parseFlatGeobufInBatches(stream, options: FlatGeobufLoaderOption
}
}

function parseFlatGeobufInBatchesToBinary(stream, options: FlatGeobufLoaderOptions) {
function parseFlatGeobufInBatchesToBinary(stream, options: ParseFlatGeobufOptions) {
// TODO: reproject binary streaming features
// const {reproject = false, _targetCrs = 'WGS84'} = (options && options.gis) || {};

const rect = options.boundingBox && convertBoundingBox(options.boundingBox);

// @ts-expect-error
const iterator = deserializeGeneric(stream, binaryFromFeature);
const iterator = deserializeGeneric(stream, binaryFromFeature, rect);
return iterator;
}

Expand All @@ -150,8 +143,8 @@ function parseFlatGeobufInBatchesToBinary(stream, options: FlatGeobufLoaderOptio
* @param options
*/
// eslint-disable-next-line complexity
async function* parseFlatGeobufInBatchesToGeoJSON(stream, options: FlatGeobufLoaderOptions) {
const {reproject = false, _targetCrs = 'WGS84'} = (options && options.gis) || {};
async function* parseFlatGeobufInBatchesToGeoJSON(stream, options: ParseFlatGeobufOptions) {
const {reproject = false, crs = 'WGS84'} = options || {};

let fgbHeader;
// let schema: Schema | undefined;
Expand All @@ -165,9 +158,9 @@ async function* parseFlatGeobufInBatchesToGeoJSON(stream, options: FlatGeobufLoa
// @ts-expect-error this looks wrong
for await (const feature of iterator) {
if (firstRecord) {
const crs = fgbHeader && fgbHeader.crs;
if (reproject && crs) {
projection = new Proj4Projection({from: crs.wkt, to: _targetCrs});
const fromCRS = fgbHeader?.crs?.wkt;
if (reproject && fromCRS) {
projection = new Proj4Projection({from: fromCRS, to: crs});
}

firstRecord = false;
Expand All @@ -181,3 +174,31 @@ async function* parseFlatGeobufInBatchesToGeoJSON(stream, options: FlatGeobufLoa
}
}
}

// HELPERS

function convertBoundingBox(boundingBox: [[number, number], [number, number]]): fgb.Rect {
return {
minX: boundingBox[0][0],
minY: boundingBox[0][1],
maxX: boundingBox[1][0],
maxY: boundingBox[1][1]
};
}

// TODO: reproject binary features
function binaryFromFeature(feature: fgb.Feature, header: fgb.HeaderMeta) {
const geometry = feature.geometry();

// FlatGeobuf files can only hold a single geometry type per file, otherwise
// GeometryType is GeometryCollection
// I believe geometry.type() is null (0) except when the geometry type isn't
// known in the header?
const geometryType = header.geometryType || geometry?.type();
const parsedGeometry = fgbToBinaryGeometry(geometry, geometryType!);
// @ts-expect-error this looks wrong
parsedGeometry.properties = parsePropertiesBinary(feature, header.columns);

// TODO: wrap binary data either in points, lines, or polygons key
return parsedGeometry;
}
4 changes: 4 additions & 0 deletions modules/loader-utils/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ export type {ImageType} from './lib/sources/utils/image-type';
export type {ImageSourceProps, ImageSourceMetadata} from './lib/sources/image-source';
export type {GetImageParameters} from './lib/sources/image-source';

export {VectorSource} from './lib/sources/vector-source';
export type {VectorSourceProps, VectorSourceMetadata} from './lib/sources/vector-source';
export type {GetFeaturesParameters} from './lib/sources/vector-source';

export type {TileSource, TileSourceProps} from './lib/sources/tile-source';
export type {TileSourceMetadata, GetTileParameters} from './lib/sources/tile-source';
export type {GetTileDataParameters} from './lib/sources/tile-source';
Expand Down

0 comments on commit 490219e

Please sign in to comment.