feat: Switch geodata providers #7393
Add build scripts
- Loading branch information
commit 7f7b69054226f1dd9d45fec502038deca1c2fb39
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| { | ||
| "resolutions": [110, 50], | ||
| "regionMapping": { | ||
| "AFE": "africa", | ||
| "AFW": "africa", | ||
| "AFR": "africa", | ||
| "AME": "americas", | ||
| "NAM": "north-america", | ||
| "LAC": "south-america", | ||
| "ASI": "asia", | ||
| "EUR": "europe", | ||
| "OCE": "oceania", | ||
| "ANT": "antarctica", | ||
| "WORLD": "world" | ||
| }, | ||
| "scopes": [ | ||
| { | ||
| "name": "africa", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [ | ||
| { | ||
| "key": "georeg", | ||
| "values": ["AFE", "AFR", "AFW"] | ||
| } | ||
| ], | ||
| "bounds": [-30, -50, 60, 50] | ||
| } | ||
| }, | ||
| { | ||
| "name": "antarctica", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [ | ||
| { | ||
| "key": "georeg", | ||
| "values": ["ANT"] | ||
| } | ||
| ], | ||
| "bounds": [] | ||
| } | ||
| }, | ||
| { | ||
| "name": "asia", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [ | ||
| { | ||
| "key": "georeg", | ||
| "values": ["ASI"] | ||
| } | ||
| ], | ||
| "bounds": [15, -90, 180, 85] | ||
| } | ||
| }, | ||
| { | ||
| "name": "europe", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [ | ||
| { | ||
| "key": "georeg", | ||
| "values": ["EUR"] | ||
| } | ||
| ], | ||
| "bounds": [-30, 0, 60, 90] | ||
| } | ||
| }, | ||
| { | ||
| "name": "north-america", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [ | ||
| { | ||
| "key": "georeg", | ||
| "values": ["AME"] | ||
| } | ||
| ], | ||
| "excludedFeatures": [ | ||
| { | ||
| "key": "intreg", | ||
| "values": ["South America"] | ||
| } | ||
| ], | ||
| "bounds": [-180, 0, -45, 85] | ||
| } | ||
| }, | ||
| { | ||
| "name": "oceania", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [ | ||
| { | ||
| "key": "georeg", | ||
| "values": ["OCE"] | ||
| } | ||
| ], | ||
| "bounds": [] | ||
| } | ||
| }, | ||
| { | ||
| "name": "south-america", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [ | ||
| { | ||
| "key": "intreg", | ||
| "values": ["South America"] | ||
| } | ||
| ], | ||
| "bounds": [-100, -70, -30, 25] | ||
| } | ||
| }, | ||
| { | ||
| "name": "world", | ||
| "specs": { | ||
| "source": "BNDA_simplified", | ||
| "acceptedFeatures": [], | ||
| "bounds": [] | ||
| } | ||
| } | ||
| ], | ||
| "simplifyTolerance": 0.1, | ||
| "outputDirGeojson": "./build/geodata/geojson", | ||
| "outputDirTopojson": "./dist/topojson", | ||
| "inputDir": "./build/geodata", | ||
| "shapefiles": ["BNDA_simplified", "GEOA_simplified", "WBYA_simplified"], | ||
| "downloadUrl": "https://geoportal.un.org/arcgis/sharing/rest/content/items/f86966528d5943efbdb83fd521dc0943/data" | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| import { exec } from 'child_process'; | ||
| import fs from 'fs'; | ||
| import { Readable } from 'stream'; | ||
| import { pipeline } from 'stream/promises'; | ||
| import config from './config.json' assert { type: 'json' }; | ||
|
|
||
| try { | ||
| // Download data from UN | ||
| const dataPath = './build/geodata'; | ||
| const outputPath = dataPath; | ||
| const filePath = `${outputPath}/geodata.zip`; | ||
|
|
||
| if (fs.existsSync(filePath)) { | ||
| console.log('Data file already exists. Skipping download.'); | ||
| } else { | ||
| console.log(`Downloading data from ${config.downloadUrl}`); | ||
| const unResponse = await fetch(config.downloadUrl); | ||
| if (!unResponse.ok || !unResponse.body) throw new Error(`Bad response: ${unResponse.status}`); | ||
|
|
||
| console.log('Processing data'); | ||
| if (!fs.existsSync(dataPath)) fs.mkdirSync(dataPath, { recursive: true }); | ||
| const file = fs.createWriteStream(filePath); | ||
| await pipeline(Readable.fromWeb(unResponse.body), file); | ||
|
|
||
| console.log(`Download complete. File saved to: ${filePath}`); | ||
| } | ||
|
|
||
| // Unzip archive | ||
| console.log('Unzipping shapefiles', `unzip -o ${filePath} -d ${outputPath}`); | ||
| // Use the shell to handle unzipping | ||
| if (!fs.existsSync(outputPath)) fs.mkdirSync(outputPath, { recursive: true }); | ||
| exec(`unzip -o ${filePath} -d ${outputPath}`); | ||
|
|
||
| console.log(`Shapefiles unzipped to ${outputPath}`); | ||
| } catch (error) { | ||
| console.error(`Error when downloading file!: ${error}`); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| import { simplify } from '@turf/simplify'; | ||
| import fs from 'fs'; | ||
| import mapshaper from 'mapshaper'; | ||
| import path from 'path'; | ||
| import config from './config.json' assert { type: 'json' }; | ||
|
|
||
| const { inputDir, resolutions, scopes, shapefiles, simplifyTolerance } = config; | ||
|
|
||
| // Create output directories | ||
| const outputDirGeojson = path.resolve(config.outputDirGeojson); | ||
| if (!fs.existsSync(outputDirGeojson)) fs.mkdirSync(outputDirGeojson, { recursive: true }); | ||
| const outputDirTopojson = path.resolve(config.outputDirTopojson); | ||
| if (!fs.existsSync(outputDirTopojson)) fs.mkdirSync(outputDirTopojson, { recursive: true }); | ||
|
|
||
| async function convertShpToGeo(filename) { | ||
| const inputFilePath = `${inputDir}/${filename}.shp`; | ||
| const outputFilePath = `${outputDirGeojson}/${filename}.geojson`; | ||
| const commands = `-i ${inputFilePath} -proj wgs84 -o format=geojson ${outputFilePath}`; | ||
| await mapshaper.runCommands(commands); | ||
|
|
||
| console.log(`GeoJSON saved to ${outputFilePath}`); | ||
| } | ||
|
|
||
| function getGeojsonFile(filename) { | ||
| console.log('📂 Loading GeoJSON file...'); | ||
| let geojson; | ||
| try { | ||
| geojson = JSON.parse(fs.readFileSync(filename, 'utf8')); | ||
| } catch (err) { | ||
| console.error(`❌ Failed to load GeoJSON input file '${filename}':`, err.message); | ||
| process.exit(1); | ||
| } | ||
|
|
||
| return geojson; | ||
| } | ||
|
|
||
| function cleanGeojson(geojson) { | ||
| return geojson.features.filter((feature) => { | ||
| const hasValidProperties = feature.properties != null; | ||
| const hasValidIso3 = feature.properties?.iso3cd !== null; | ||
camdecoster marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| const hasValidGeometry = feature.geometry != null; | ||
|
|
||
| // Remove Hawaii with specific globalid | ||
| const isHawaiiOverlap = feature.properties?.globalid === '{8B42E894-6AF5-4236-B04D-8F634A159724}'; | ||
camdecoster marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| return hasValidProperties && hasValidIso3 && hasValidGeometry && !isHawaiiOverlap; | ||
| }); | ||
| } | ||
|
|
||
| function getScopedFeatures(geojson, acceptedFeatures, excludedFeatures) { | ||
| if (!acceptedFeatures.length) return geojson; | ||
|
|
||
| return geojson.filter((feature) => { | ||
| const hasAcceptedValue = acceptedFeatures.some(({ key, values }) => | ||
| values.includes(feature?.properties?.[key]) | ||
| ); | ||
| const hasExcludedValue = excludedFeatures.some(({ key, values }) => | ||
| values.includes(feature?.properties?.[key]) | ||
| ); | ||
|
|
||
| return hasAcceptedValue && !hasExcludedValue; | ||
| }); | ||
| } | ||
|
|
||
| function saveGeojson(region, resolution, layer, geojson) { | ||
| try { | ||
| const regionDir = path.join(outputDirGeojson, `${region}_${resolution}m`); | ||
| const filePath = path.join(regionDir, `${layer}.geojson`); | ||
| if (!fs.existsSync(regionDir)) fs.mkdirSync(regionDir, { recursive: true }); | ||
| fs.writeFileSync(filePath, JSON.stringify(geojson)); | ||
| console.log(`🌐 Saved: ${filePath}`); | ||
| return filePath; | ||
| } catch (err) { | ||
| console.error(`❌ Failed to save Geojson for ${region} (${resolution}m):`, err.message); | ||
| } | ||
| } | ||
|
|
||
| async function createCountriesLayer({ acceptedFeatures, excludedFeatures, name, source }) { | ||
| console.log(`Building countries layer for '${name}'`); | ||
| const geojson = getGeojsonFile(`${outputDirGeojson}/${source}.geojson`); | ||
| const cleanedFeatures = cleanGeojson(geojson); | ||
| const scopedFeatures = getScopedFeatures(cleanedFeatures, acceptedFeatures, excludedFeatures); | ||
|
|
||
| const featureCollection = { type: 'FeatureCollection', features: scopedFeatures }; | ||
| const layer = 'countries'; | ||
| // TODO: Verify resolution of UN geodata | ||
| // Save 50m resolution (raw) | ||
| saveGeojson(name, 50, layer, featureCollection); | ||
|
|
||
| // TODO: Verify tolerance to match 110m resolution | ||
| // Save 110m resolution (simplified) | ||
| const simplifiedFeatures = featureCollection.features.map((f) => | ||
| simplify(f, { tolerance: simplifyTolerance, highQuality: true }) | ||
| ); | ||
| const simplifiedCollection = { type: 'FeatureCollection', features: simplifiedFeatures }; | ||
| saveGeojson(name, 110, layer, simplifiedCollection); | ||
| } | ||
|
|
||
| async function createLandLayer(name) { | ||
| console.log(`Building land layer for '${name}'`); | ||
| for (const resolution of resolutions) { | ||
| const inputFilePath = `${outputDirGeojson}/${name}_${resolution}m/countries.geojson`; | ||
| const outputFilePath = `${outputDirGeojson}/${name}_${resolution}m/land.geojson`; | ||
| const commands = `${inputFilePath} -dissolve -o ${outputFilePath}`; | ||
| await mapshaper.runCommands(commands); | ||
| } | ||
| } | ||
|
|
||
| async function createCoastlinesLayer(name) { | ||
| console.log(`Building coastlines layer for '${name}'`); | ||
| for (const resolution of resolutions) { | ||
| const inputFilePath = `${outputDirGeojson}/${name}_${resolution}m/countries.geojson`; | ||
| const outputFilePath = `${outputDirGeojson}/${name}_${resolution}m/coastlines.geojson`; | ||
| const commands = `${inputFilePath} -dissolve -lines -o ${outputFilePath}`; | ||
| await mapshaper.runCommands(commands); | ||
| } | ||
| } | ||
|
|
||
| async function createOceansLayer({ bounds, name }) { | ||
| console.log(`Building oceans layer for '${name}'`); | ||
| for (const resolution of resolutions) { | ||
| const inputFilePath = `./tasks/topojson/world_rectangle.geojson`; | ||
| const outputFilePath = `${outputDirGeojson}/${name}_${resolution}m/oceans.geojson`; | ||
| const eraseFilePath = `${outputDirGeojson}/GEOA_simplified.geojson`; | ||
| const commands = `${inputFilePath} ${bounds.length ? `-clip bbox=${bounds.join(',')}` : ''} -erase ${eraseFilePath} -o ${outputFilePath}`; | ||
| await mapshaper.runCommands(commands); | ||
| } | ||
| } | ||
|
|
||
| async function createWaterbodiesLayer({ bounds, name }) { | ||
| // Clip the waterbodies shapefile to each continent | ||
| console.log(`Building waterbodies layer for '${name}'`); | ||
| for (const resolution of resolutions) { | ||
| const inputFilePath = `${outputDirGeojson}/WBYA_simplified.geojson`; | ||
| const outputFilePath = `${outputDirGeojson}/${name}_${resolution}m/waterbodies.geojson`; | ||
| const commands = `${inputFilePath} ${bounds.length ? `-clip bbox=${bounds.join(',')}` : ''} -o ${outputFilePath}`; | ||
| await mapshaper.runCommands(commands); | ||
| } | ||
| } | ||
|
|
||
| async function combineFiles() { | ||
| for (const resolution of resolutions) { | ||
| for (const { name } of scopes) { | ||
| const regionDir = path.join(outputDirGeojson, `${name}_${resolution}m`); | ||
| if (!fs.existsSync(regionDir)) { | ||
| console.log(`Couldn't find ${regionDir}`); | ||
| continue; | ||
| } | ||
|
|
||
| const outputFile = `${outputDirTopojson}/${name}_${resolution}m.json`; | ||
| // Layer names default to file names | ||
| const commands = `-i ${regionDir}/*.geojson combine-files -o format=topojson ${outputFile}`; | ||
| await mapshaper.runCommands(commands); | ||
|
|
||
| console.log(`Topojson saved to: ${outputFile}`); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| for (const shapefile of shapefiles) { | ||
| await convertShpToGeo(shapefile); | ||
| } | ||
| for (const { | ||
| name, | ||
| specs: { acceptedFeatures, bounds, excludedFeatures = [], source } | ||
| } of scopes) { | ||
| await createCountriesLayer({ acceptedFeatures, excludedFeatures, name, source }); | ||
| await createLandLayer(name); | ||
| await createCoastlinesLayer(name); | ||
| await createOceansLayer({ bounds, name }); | ||
| await createWaterbodiesLayer({ bounds, name }); | ||
| } | ||
|
|
||
| await combineFiles(); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "type": "FeatureCollection", | ||
| "features": [ | ||
| { | ||
| "type": "Feature", | ||
| "properties": {}, | ||
| "geometry": { | ||
| "type": "Polygon", | ||
| "coordinates": [ | ||
| [ | ||
| [-180, -90], | ||
| [180, -90], | ||
| [180, 90], | ||
| [-180, 90], | ||
| [-180,-90] | ||
| ] | ||
| ] | ||
| } | ||
| } | ||
| ] | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As these origin URLs may change (link rot), I worry that this piece of the script could become out of date quickly. I'm not sure there's too much we can do here but I might recommend committing the current source file (but not adding it to the
dist/. Maybe someone else has a better idea.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That seems reasonable. The script could be updated to look for the file first and only download if it doesn't exist.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@marthacryan do you have an opinion or ideas on this? What's the ideal dev UX here?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
FYI, in
3b47c67I added the archive from the UN.