From c13b320c5237e5ce1b95b98d41d0e1e6edfd75c2 Mon Sep 17 00:00:00 2001
From: Chen Wang <cwang138@illinois.edu>
Date: Wed, 10 Jan 2024 12:02:53 -0600
Subject: [PATCH] cleanup script

---
 .../cleanup_dataset_format.ipynb              | 161 ++++++++++++++++++
 1 file changed, 161 insertions(+)
 create mode 100644 mongodb-field-cleanup/cleanup_dataset_format.ipynb

diff --git a/mongodb-field-cleanup/cleanup_dataset_format.ipynb b/mongodb-field-cleanup/cleanup_dataset_format.ipynb
new file mode 100644
index 0000000..e4b2d66
--- /dev/null
+++ b/mongodb-field-cleanup/cleanup_dataset_format.ipynb
@@ -0,0 +1,161 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9b9b5b08-6598-46d2-be2f-8156d51e7c7d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymongo import MongoClient\n",
+    "import getpass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19ddaa2a-2faf-4e33-a85e-d67d6505b644",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mongo_username = input(\"Enter MongoDB username: \")\n",
+    "mongo_password = getpass.getpass(\"Enter MongoDB password: \")\n",
+    "host = input(\"Enter MongoDB host (default is 'localhost'): \") or \"localhost\"\n",
+    "port = input(\"Enter MongoDB port (default is '27017'): \") or \"27017\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ffd69267-c8c6-42e0-8f5f-f46d73a4f342",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = MongoClient('mongodb://%s:%s@%s:%s' % (mongo_username, mongo_password, host, port))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b5b9aec7-19df-47ca-a100-6c3a25367725",
+   "metadata": {},
+   "source": [
+    "### Summarize what formats we have"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "4c5b2ad8-96e5-46f3-a153-05d5c1acb0e1",
+   "metadata": {},
+   "source": [
+    "for space in client['spacedb']['Space'].find():\n",
+    "    if space[\"metadata\"][\"name\"] == \"incore\":\n",
+    "        incore_id = space[\"_id\"]\n",
+    "    if space[\"metadata\"][\"name\"] == \"ergo\":\n",
+    "        ergo_id = space[\"_id\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1024abd1-2dbb-430f-af36-a1f8c3eed9df",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "unique_format = {}\n",
+    "for dataset in client['datadb']['Dataset'].find():\n",
+    "    if \"format\" not in dataset or dataset[\"format\"]==\"\" or dataset[\"format\"]==None:\n",
+    "        print(dataset['_id'], \"missing format.\")\n",
+    "    else:\n",
+    "        if dataset[\"format\"] not in unique_format.keys():\n",
+    "            unique_format[dataset[\"format\"]] = 1\n",
+    "        else:\n",
+    "            unique_format[dataset[\"format\"]] += 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0084fbcc-63fc-4ab7-8a42-f72187748edc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "unique_format"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "473e5c85-72ea-4b6d-8d54-1d228e72790e",
+   "metadata": {},
+   "source": [
+    "### Consolidate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26c6d33d-6270-4991-a7ca-b04e8ad1dc16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mapping = {\n",
+    "    \"table\":\"table\",\n",
+    "    \"csv\": \"table\",\n",
+    "    \"shapefile\":\"shapefile\",\n",
+    "    \"raster\": \"raster\",\n",
+    "    \"geotif\": \"raster\",\n",
+    "    \"geotiff\": \"raster\",\n",
+    "    \"shp-network\":\"shp-network\",\n",
+    "    \"textfiles\": \"textfile\",\n",
+    "    \"json\": \"json\",\n",
+    "    \"unknown\": \"unknown\",\n",
+    "    \"inp\": \"unknown\",\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "210d5656-9d0c-4133-9d58-a1c22615da3b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for dataset in client['datadb']['Dataset'].find():\n",
+    "    if \"format\" not in dataset or dataset[\"format\"]==\"\" or dataset[\"format\"]==None:\n",
+    "        dataset['format'] = \"unknown\"\n",
+    "    else:\n",
+    "       dataset['format'] = mapping[dataset['format'].lower()]\n",
+    "    # uncomment to deposit change to database \n",
+    "    # client['datadb']['Dataset'].replace_one({'_id':dataset['_id']}, dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a7bf335-d3c8-4963-b377-95a20633a7bd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}