Skip to content

Update [Notebook] : Add bypass function to directly download outputsfrom execution name + add check on each step to verify needed files are here #59

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Albireo-B wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Update [Notebook] : Add bypass function to directly download outputsfrom execution name + add check on each step to verify needed files are here #59

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 92 additions & 1 deletion examples/freesurfer/freesurfer_longitudinal_processing/freesurfer_longitudinal.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
"\n",
"# VIP\n",
"from vip_client import VipSession\n",
"from vip_client.utils import vip\n",
"\n",
"def flatten_folder(folder):\n",
" parent = folder.parent\n",
Expand Down Expand Up @@ -120,7 +121,31 @@
" parent_type = \"folder\"\n",
" current = next_path\n",
"\n",
" return resource"
" return resource\n",
"\n",
"def download_by_session_name(session_name, local_dir):\n",
" \"\"\"\n",
" Download outputs from a session by name only.\n",
" Only works with the 10 last executions.\n",
Comment thread
camarasu marked this conversation as resolved.
" \"\"\"\n",
" local_dir = Path(local_dir)\n",
" local_dir.mkdir(parents=True, exist_ok=True)\n",
" # Find workflows matching session name\n",
" workflows = [e[\"identifier\"] for e in vip.list_executions() if e.get(\"name\") == session_name]\n",
" print(f\"Found {len(workflows)} workflow(s)\")\n",
"\n",
" for wid in workflows:\n",
" files = [(f[\"path\"], str(local_dir / Path(f[\"path\"]).name))\n",
" for f in vip.get_exec_results(wid)\n",
" if not f.get(\"isDirectory\")]\n",
"\n",
" for (vip_path, local_path), success in vip.download_parallel(files):\n",
" print(f\"{'✓' if success else '✗'} {Path(vip_path).name}\")\n",
"\n",
" print(f\"Saved to: {local_dir}\")\n",
"\n",
"def is_empty(path: Path) -> bool:\n",
" return not any(p.name != \"session_data.json\" for p in path.iterdir())"
],
"id": "18aa877afac9a865",
"outputs": [],
Expand Down Expand Up @@ -343,6 +368,9 @@
"metadata": {},
"cell_type": "code",
"source": [
"if not cfg.LOCAL_DATASET_PATH.exists() or is_empty(cfg.LOCAL_DATASET_PATH):\n",
" raise FileNotFoundError(f\"The local dataset path {cfg.LOCAL_DATASET_PATH} does not exist or is empty.\")\n",
"\n",
"# Remove FreeSurfer CROSS output directory if existing\n",
"if FS_CROSS_OUTPUT_DIR.exists():\n",
" shutil.rmtree(FS_CROSS_OUTPUT_DIR)\n",
Expand Down Expand Up @@ -412,6 +440,23 @@
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "markdown",
"source": "⚠️ Use this last cell only if the previous one didn't work. It directly downloads the outputs from a session by a name you provide. You can find the session name on the vip portal as execution name, or in the CROSS/Outputs/session_data.json file.",
"id": "b2c407bdb11858d6"
},
{
"metadata": {},
"cell_type": "code",
"source": [
"# Download outputs from a session by name only\n",
"download_by_session_name(\"SESSION_NAME\", str(FS_CROSS_OUTPUT_DIR))"
],
"id": "d32ca116a43aeb15",
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "markdown",
Expand Down Expand Up @@ -442,6 +487,9 @@
"metadata": {},
"cell_type": "code",
"source": [
"if not FS_CROSS_OUTPUT_DIR.exists() or is_empty(FS_CROSS_OUTPUT_DIR):\n",
Comment thread
camarasu marked this conversation as resolved.
" raise FileNotFoundError(f\"The CROSS output directory {FS_CROSS_OUTPUT_DIR} does not exist or is empty.\")\n",
"\n",
"# Remove FreeSurfer BASE inputs/outputs directories if existing\n",
"if FS_BASE_INPUT_DIR.exists():\n",
" shutil.rmtree(FS_BASE_INPUT_DIR)\n",
Expand Down Expand Up @@ -525,6 +573,9 @@
"metadata": {},
"cell_type": "code",
"source": [
"if not FS_BASE_INPUT_DIR.exists() or is_empty(FS_BASE_INPUT_DIR):\n",
" raise FileNotFoundError(f\"The BASE input directory {FS_BASE_INPUT_DIR} does not exist or is empty.\")\n",
"\n",
"# Collect tarballs and BASE_IDs\n",
"tp_tarballs = [f for f in FS_BASE_INPUT_DIR.iterdir() if f.suffix in (\".tgz\", \".tar.gz\") and \"_TPs\" in f.name]\n",
"base_ids = [f.name.split(\"_\")[0] for f in tp_tarballs]\n",
Expand Down Expand Up @@ -579,6 +630,23 @@
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "markdown",
"source": "⚠️ Use this last cell only if the previous one didn't work. It directly downloads the outputs from a session by a name you provide. You can find the session name on the vip portal as execution name, or in the BASE/Outputs/session_data.json file.",
"id": "40677f8b4ddcc15c"
},
{
"metadata": {},
"cell_type": "code",
"source": [
"# Download outputs from a session by name only\n",
"download_by_session_name(\"SESSION_NAME\", str(FS_BASE_OUTPUT_DIR))"
],
"id": "1172112f1fafb734",
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "markdown",
Expand Down Expand Up @@ -612,6 +680,9 @@
"metadata": {},
"cell_type": "code",
"source": [
"if not FS_BASE_OUTPUT_DIR.exists() or is_empty(FS_BASE_OUTPUT_DIR):\n",
" raise FileNotFoundError(f\"The BASE output directory {FS_BASE_OUTPUT_DIR} does not exist or is empty.\")\n",
"\n",
"# Remove FreeSurfer LONG output directory if existing\n",
"if FS_LONG_OUTPUT_DIR.exists():\n",
" shutil.rmtree(FS_LONG_OUTPUT_DIR)\n",
Expand Down Expand Up @@ -692,6 +763,23 @@
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "markdown",
"source": "⚠️ Use this last cell only if the previous one didn't work. It directly downloads the outputs from a session by a name you provide. You can find the session name on the vip portal as execution name, or in the LONG/Outputs/session_data.json file.",
"id": "6841fad83b295709"
},
{
"metadata": {},
"cell_type": "code",
"source": [
"# Download outputs from a session by name only\n",
"download_by_session_name(\"SESSION_NAME\", str(FS_LONG_OUTPUT_DIR))"
],
"id": "3a8a884613a57a15",
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "markdown",
Expand All @@ -718,6 +806,9 @@
"metadata": {},
"cell_type": "code",
"source": [
"if not FS_LONG_OUTPUT_DIR.exists() or is_empty(FS_LONG_OUTPUT_DIR):\n",
" raise FileNotFoundError(f\"The LONG output directory {FS_LONG_OUTPUT_DIR} does not exist or is empty.\")\n",
"\n",
"# Restore all session_data.json and licenses files\n",
"def restore_files(files):\n",
" for original_path, temp_path in files:\n",
Expand Down
Loading