Spaces:
Sleeping
Sleeping
File size: 5,074 Bytes
7289e5d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
{
"cells": [
{
"cell_type": "markdown",
"id": "6e29e27b-6baa-40ac-bbf8-230da2f94d17",
"metadata": {},
"source": [
"# Rounding data\n",
"\n",
"This should've been in the original preprocessing script, but the decision to round the data came much later so I'm including this as an addition."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "81f3a57e-fa63-43b4-90f9-407f03abae79",
"metadata": {},
"outputs": [],
"source": [
"import ibis\n",
"from ibis import _\n",
"import ibis.selectors as s\n",
"parquet = \"https://huggingface.co/spaces/boettiger-lab/pad-us/resolve/575a4505f3eb1703070977d9d26f6a770045309c/pad-stats.parquet\"\n",
"con = ibis.duckdb.connect(extensions=[\"spatial\"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b4745567-45db-406c-9b08-d72a97908d04",
"metadata": {},
"outputs": [],
"source": [
"#rounding data with ibis\n",
"us = (con\n",
" .read_parquet(parquet)\n",
" .cast({\"geometry\": \"geometry\"})\n",
" .mutate(geometry=_.geometry.convert(\n",
" \"+proj=moll +lon_0=0 +datum=WGS84 +units=m +no_defs\",\n",
" \"epsg:4326\"\n",
" ))\n",
" .mutate(richness=_.richness.round(3),\n",
" rsr=_.rsr.round(3),\n",
" all_species_rwr=_.all_species_rwr.round(3),\n",
" all_species_richness=_.all_species_richness.round(3),\n",
" manageable_carbon=_.manageable_carbon.round(3),\n",
" irrecoverable_carbon = _.irrecoverable_carbon.round(3),\n",
" human_impact=_.human_impact.round(3),\n",
" deforest_carbon=_.deforest_carbon.round(3),\n",
" biodiversity_intactness_loss=_.biodiversity_intactness_loss.round(3),\n",
" forest_integrity_loss=_.forest_integrity_loss.round(3),\n",
" crop_reduction =_.crop_reduction.round(3), \n",
" crop_expansion =_.crop_expansion.round(3)\n",
" )\n",
" )"
]
},
{
"cell_type": "markdown",
"id": "5985e893-ed21-487c-a609-e449edae9012",
"metadata": {},
"source": [
"# Save as PMTiles + Upload data"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "100db9ae-e167-45ed-8c44-6205e5630923",
"metadata": {},
"outputs": [],
"source": [
"import subprocess\n",
"import os\n",
"from huggingface_hub import HfApi, login\n",
"import streamlit as st\n",
"\n",
"login(st.secrets[\"HF_TOKEN\"])\n",
"api = HfApi()\n",
"\n",
"def hf_upload(file, repo_id,repo_type):\n",
" info = api.upload_file(\n",
" path_or_fileobj=file,\n",
" path_in_repo=file,\n",
" repo_id=repo_id,\n",
" repo_type=repo_type,\n",
" )\n",
"def generate_pmtiles(input_file, output_file, max_zoom=12):\n",
" # Ensure Tippecanoe is installed\n",
" if subprocess.call([\"which\", \"tippecanoe\"], stdout=subprocess.DEVNULL) != 0:\n",
" raise RuntimeError(\"Tippecanoe is not installed or not in PATH\")\n",
"\n",
" # Construct the Tippecanoe command\n",
" command = [\n",
" \"tippecanoe\",\n",
" \"-o\", output_file,\n",
" \"-zg\",\n",
" \"--extend-zooms-if-still-dropping\",\n",
" \"--force\",\n",
" \"--projection\", \"EPSG:4326\", \n",
" \"-L\",\"pad-stats:\"+input_file,\n",
" ]\n",
" # Run Tippecanoe\n",
" try:\n",
" subprocess.run(command, check=True)\n",
" print(f\"Successfully generated PMTiles file: {output_file}\")\n",
" except subprocess.CalledProcessError as e:\n",
" print(f\"Error running Tippecanoe: {e}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eaf1a1cf-a5db-462c-a257-c68547b35d4d",
"metadata": {},
"outputs": [],
"source": [
"gdf = us.execute().set_crs(\"EPSG:4326\")\n",
"\n",
"gdf.to_file(\"pad-stats.geojson\")\n",
"generate_pmtiles(\"pad-stats.geojson\", \"pad-stats.pmtiles\")\n",
"hf_upload(\"pad-stats.pmtiles\", \"boettiger-lab/pad-us-3\", \"dataset\")\n",
"\n",
"gdf.to_parquet(\"pad-stats.parquet\")\n",
"hf_upload(\"pad-stats.parquet\", \"boettiger-lab/pad-us-3\", \"dataset\")\n",
"hf_upload(\"pad-stats.parquet\", \"boettiger-lab/pad-us\", \"space\") # redundant but I want a local copy for testing\n",
"\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
|