From adc4d82d7cc5b784eb06c873c28fcc72643fe5b9 Mon Sep 17 00:00:00 2001 From: Will Mooreston Date: Tue, 1 Sep 2026 18:03:37 -0700 Subject: [PATCH 1/9] Kanban 1743: add a remote R script engine backed by an HTTP runner Runs R in a remote container instead of on the appserver. The working directory is tarred, staged to object storage, and handed to the runner as presigned URLs, so the runner holds no AWS credentials and no LabKey API key: each URL is scoped to one object and expires. DirectoryArchive lifts the tar create and extract logic out of DockerServiceImpl's private methods so it can be reused over a transport other than the Docker API. Extraction rejects entries resolving outside the destination, since the result tar comes back from a container that ran customer script code. Unit tests cover binary round-trip, input filtering and path traversal. RRemoteScriptEngine mirrors RDockerScriptEngine: the engine maps paths and delegates, the transport lives in an injected service registered by cloudServices. ExternalScriptEngineDefinition gains isRemoteRunner() to separate an HTTP runner from an Rserve host, since both are "remote". It rides the existing JSON configuration blob, so no schema change is needed. ScriptEngineManagerImpl throws rather than returning null when a definition targets a runner that is not configured, because returning null would silently fall back to running on the appserver, which is what this engine exists to avoid. --- .../api/remoterunner/RemoteRunnerService.java | 46 ++++ .../ExternalScriptEngineDefinition.java | 7 + .../reports/report/r/RRemoteScriptEngine.java | 136 +++++++++++ .../report/r/RRemoteScriptEngineFactory.java | 28 +++ .../org/labkey/api/util/DirectoryArchive.java | 221 ++++++++++++++++++ core/src/org/labkey/core/CoreModule.java | 1 + .../ExternalScriptEngineDefinitionImpl.java | 26 +++ .../core/reports/ScriptEngineManagerImpl.java | 16 ++ 8 files changed, 481 insertions(+) create mode 100644 api/src/org/labkey/api/remoterunner/RemoteRunnerService.java create mode 100644 api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java create mode 100644 api/src/org/labkey/api/reports/report/r/RRemoteScriptEngineFactory.java create mode 100644 api/src/org/labkey/api/util/DirectoryArchive.java diff --git a/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java b/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java new file mode 100644 index 00000000000..c7b4bd7188c --- /dev/null +++ b/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 LabKey Corporation. All rights reserved. No portion of this work may be reproduced + * in any form or by any electronic or mechanical means without written permission from LabKey Corporation. + */ +package org.labkey.api.remoterunner; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.vfs.FileLike; + +import java.io.FileFilter; +import java.io.IOException; + +/** + * Runs a script in a remote container over HTTP. + * + * The working directory is staged to object storage and handed to the runner as presigned URLs, so the runner holds no + * credentials: each URL is scoped to a single object and expires. Implemented by the cloudServices module. + */ +public interface RemoteRunnerService +{ + static @Nullable RemoteRunnerService get() + { + return ServiceRegistry.get().getService(RemoteRunnerService.class); + } + + static void setInstance(RemoteRunnerService impl) + { + ServiceRegistry.get().registerService(RemoteRunnerService.class, impl); + } + + /** True when a runner endpoint and a staging bucket are both configured. */ + boolean isEnabled(); + + /** + * Tar {@code localWorkingDir}, run {@code scriptFile} against it in the remote runner, and unpack the result back + * over {@code localWorkingDir}. + * + * @param scriptFile the script to run, already written into the working directory + * @param localWorkingDir working directory on this server + * @param remoteWorkingDir path the runner will see, used for tar entry names and path mapping + * @param inputFiles which files in the working directory to send + */ + void executeR(FileLike scriptFile, String localWorkingDir, String remoteWorkingDir, @Nullable FileFilter inputFiles) + throws IOException; +} diff --git a/api/src/org/labkey/api/reports/ExternalScriptEngineDefinition.java b/api/src/org/labkey/api/reports/ExternalScriptEngineDefinition.java index ab489dc7b03..c8590508b03 100644 --- a/api/src/org/labkey/api/reports/ExternalScriptEngineDefinition.java +++ b/api/src/org/labkey/api/reports/ExternalScriptEngineDefinition.java @@ -83,6 +83,13 @@ enum Type void setDocker(boolean docker); boolean isDocker(); + /** + * True when a remote definition targets the HTTP script runner rather than Rserve. Both are "remote", so + * this is what separates them. + */ + void setRemoteRunner(boolean remoteRunner); + boolean isRemoteRunner(); + void setDockerImageRowId(Integer rowId); Integer getDockerImageRowId(); String getDockerImageConfig(); diff --git a/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java new file mode 100644 index 00000000000..989b78410af --- /dev/null +++ b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2026 LabKey Corporation. All rights reserved. No portion of this work may be reproduced + * in any form or by any electronic or mechanical means without written permission from LabKey Corporation. + */ +package org.labkey.api.reports.report.r; + +import org.apache.commons.io.FilenameUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.miniprofiler.CustomTiming; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.pipeline.file.PathMapper; +import org.labkey.api.pipeline.file.PathMapperImpl; +import org.labkey.api.query.ValidationException; +import org.labkey.api.remoterunner.RemoteRunnerService; +import org.labkey.api.reports.ExternalScriptEngineDefinition; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.UnexpectedException; +import org.labkey.vfs.FileLike; + +import javax.script.ScriptContext; +import javax.script.ScriptException; +import java.io.File; +import java.io.FileFilter; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; + +/** + * Runs R in a remote container reached over HTTP, with the working directory staged through object storage. + * + * Shaped after {@link RDockerScriptEngine}: the engine only maps paths and delegates, the transport lives in the + * injected service. + */ +public class RRemoteScriptEngine extends RScriptEngine +{ + /** Fixed because the runner unpacks the working directory to the same place every run. */ + public static final String REMOTE_WORKING_DIR = "/work"; + + private final RemoteRunnerService _service; + + public RRemoteScriptEngine(@NotNull ExternalScriptEngineDefinition def, @Nullable RemoteRunnerService service) + { + super(def); + _service = service; + + def.setPathMapper(new PathMapperImpl() + { + void setMapping() + { + String wd = getWorkingDir(getContext()).toNioPathForRead().toFile().getAbsolutePath() + .replace("\\", "/").replace("/./", "/"); + super.setPathMap(Collections.singletonMap( + REMOTE_WORKING_DIR, + new File(wd).toURI().toString())); + } + + @Override + public String remoteToLocal(String remoteURI) + { + setMapping(); + return super.remoteToLocal(remoteURI); + } + + @Override + public String localToRemote(String localURI) + { + setMapping(); + return super.localToRemote(localURI); + } + + @Override + public ValidationException getValidationErrors() + { + setMapping(); + return super.getValidationErrors(); + } + }); + } + + @Override + protected Object eval(FileLike scriptFile, ScriptContext context) throws ScriptException + { + if (null == _service) + throw new ScriptException("Script evaluation attempted with no RemoteRunnerService instance available."); + + StringBuffer output = new StringBuffer(); + try (CustomTiming t = MiniProfiler.custom("remoteRunner", "execute r in remote runner")) + { + _service.executeR(scriptFile, getRWorkingDir(context), REMOTE_WORKING_DIR, inputFiles()); + appendConsoleOutput(context, output); + } + catch (Exception e) + { + throw new ScriptException("An error occurred when running the script '" + scriptFile.getName() + + "', msg " + e.getMessage() + ").\n" + e); + } + + String scriptOut = output.toString(); + // R CMD BATCH writes to .Rout rather than failing the process, so the only error signal is in the output. + if (scriptOut.contains("Execution halted")) + throw new ScriptException("An error occurred when running the script '" + scriptFile.getName() + "'.\n" + scriptOut); + return scriptOut; + } + + private static FileFilter inputFiles() + { + return pathname -> + pathname.isFile() && + (RScriptEngineFactory.isRScriptEngine(new String[]{FilenameUtils.getExtension(pathname.getName())}) + || RReport.DATA_INPUT.equals(pathname.getName())); + } + + @Override + public String getRemotePath(FileLike localFile) + { + URI localUri = FileUtil.getAbsoluteCaseSensitiveFile(localFile).toURI(); + URI remote = RserveScriptEngine.makeLocalToRemotePath(_def, getWorkingDir(getContext()), localUri); + return PathMapper.uriToPath(remote); + } + + @Override + public String getRemotePath(String local) + { + try + { + URI localUri = PathMapper.pathToUri(local); + URI remote = RserveScriptEngine.makeLocalToRemotePath(_def, getWorkingDir(getContext()), localUri); + return PathMapper.uriToPath(remote); + } + catch (URISyntaxException e) + { + throw UnexpectedException.wrap(e); + } + } +} diff --git a/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngineFactory.java b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngineFactory.java new file mode 100644 index 00000000000..a2c8bfab643 --- /dev/null +++ b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngineFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2026 LabKey Corporation. All rights reserved. No portion of this work may be reproduced + * in any form or by any electronic or mechanical means without written permission from LabKey Corporation. + */ +package org.labkey.api.reports.report.r; + +import org.labkey.api.remoterunner.RemoteRunnerService; +import org.labkey.api.reports.ExternalScriptEngineDefinition; +import org.labkey.api.reports.ExternalScriptEngineFactory; + +import javax.script.ScriptEngine; + +public class RRemoteScriptEngineFactory extends ExternalScriptEngineFactory +{ + public RRemoteScriptEngineFactory(ExternalScriptEngineDefinition def) + { + super(def); + } + + @Override + public synchronized ScriptEngine getScriptEngine() + { + RemoteRunnerService service = RemoteRunnerService.get(); + if (null != service && service.isEnabled()) + return new RRemoteScriptEngine(_def, service); + return null; + } +} diff --git a/api/src/org/labkey/api/util/DirectoryArchive.java b/api/src/org/labkey/api/util/DirectoryArchive.java new file mode 100644 index 00000000000..dbee116dd89 --- /dev/null +++ b/api/src/org/labkey/api/util/DirectoryArchive.java @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2026 LabKey Corporation. All rights reserved. No portion of this work may be reproduced + * in any form or by any electronic or mechanical means without written permission from LabKey Corporation. + */ +package org.labkey.api.util; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.io.IOUtils; +import org.jetbrains.annotations.Nullable; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Tar a working directory and untar it again. Equivalent to the archive handling inside DockerServiceImpl, lifted out + * so remote script execution can reuse it over transports other than the Docker API. + */ +public class DirectoryArchive +{ + private DirectoryArchive() + { + } + + /** Tar {@code directory} into {@code target}. Entries are named relative to {@code parentName}. */ + public static void create(File directory, @Nullable FileFilter filter, String parentName, File target) throws IOException + { + try (OutputStream fos = new FileOutputStream(target); + TarArchiveOutputStream tar = new TarArchiveOutputStream(fos)) + { + // Long working-directory paths are routine, and the default format silently truncates past 100 chars. + tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + addDirectory(tar, directory, filter, trimTrailingSlash(parentName)); + tar.finish(); + } + } + + private static void addDirectory(TarArchiveOutputStream tar, File dir, @Nullable FileFilter filter, String prefix) throws IOException + { + File[] children = dir.listFiles(); + if (children == null) + return; + + for (File child : children) + { + String entryName = prefix + "/" + child.getName(); + if (child.isDirectory()) + { + addDirectory(tar, child, filter, entryName); + continue; + } + if (filter != null && !filter.accept(child)) + continue; + + TarArchiveEntry entry = new TarArchiveEntry(child, entryName); + tar.putArchiveEntry(entry); + try (InputStream in = new FileInputStream(child)) + { + IOUtils.copy(in, tar); + } + tar.closeArchiveEntry(); + } + } + + /** + * Untar {@code in} beneath {@code destination}, stripping the leading path component so a tar made with + * {@code parentName} unpacks flat into the destination. + * + * Entries resolving outside the destination are rejected. The tar arrives from a remote runner that executed + * customer script code, so its entry names are untrusted input. + */ + public static void extract(InputStream in, File destination) throws IOException + { + Path root = destination.toPath().toAbsolutePath().normalize(); + + try (TarArchiveInputStream tar = new TarArchiveInputStream(new BufferedInputStream(in))) + { + TarArchiveEntry entry; + while ((entry = tar.getNextEntry()) != null) + { + String name = stripLeadingComponent(entry.getName()); + if (name.isEmpty()) + continue; + + Path resolved = root.resolve(name).normalize(); + if (!resolved.startsWith(root)) + throw new IOException("Refusing to extract entry outside the destination directory: " + entry.getName()); + + File target = resolved.toFile(); + if (entry.isDirectory()) + { + FileUtil.mkdirs(target); + continue; + } + + FileUtil.mkdirs(target.getParentFile()); + try (OutputStream out = new FileOutputStream(target)) + { + IOUtils.copy(tar, out); + } + } + } + } + + private static String stripLeadingComponent(String entryName) + { + String name = entryName.startsWith("/") ? entryName.substring(1) : entryName; + int slash = name.indexOf('/'); + return slash < 0 ? "" : name.substring(slash + 1); + } + + private static String trimTrailingSlash(String s) + { + return s.endsWith("/") ? s.substring(0, s.length() - 1) : s; + } + + public static class TestCase extends org.junit.Assert + { + @org.junit.Test + public void roundTripPreservesContentAndBinary() throws IOException + { + File src = FileUtil.createTempDirectory("arch-src").toFile(); + File dest = FileUtil.createTempDirectory("arch-dest").toFile(); + try + { + Files.writeString(new File(src, "script.R").toPath(), "cat(\"hi\")"); + File nested = new File(src, "sub"); + FileUtil.mkdirs(nested); + byte[] binary = new byte[]{0, 1, 2, (byte) 0xFF, 0x7F}; + Files.write(new File(nested, "plot.png").toPath(), binary); + + File tar = File.createTempFile("arch", ".tar"); + create(src, null, "/work", tar); + try (InputStream in = new FileInputStream(tar)) + { + extract(in, dest); + } + + assertEquals("cat(\"hi\")", Files.readString(new File(dest, "script.R").toPath())); + assertArrayEquals(binary, Files.readAllBytes(new File(dest, "sub/plot.png").toPath())); + } + finally + { + FileUtil.deleteDir(src); + FileUtil.deleteDir(dest); + } + } + + @org.junit.Test + public void filterExcludesUnwantedFiles() throws IOException + { + File src = FileUtil.createTempDirectory("arch-filter").toFile(); + File dest = FileUtil.createTempDirectory("arch-filter-out").toFile(); + try + { + Files.writeString(new File(src, "keep.R").toPath(), "keep"); + Files.writeString(new File(src, "drop.tmp").toPath(), "drop"); + + File tar = File.createTempFile("arch", ".tar"); + create(src, f -> f.getName().endsWith(".R"), "/work", tar); + try (InputStream in = new FileInputStream(tar)) + { + extract(in, dest); + } + + assertTrue(new File(dest, "keep.R").exists()); + assertFalse(new File(dest, "drop.tmp").exists()); + } + finally + { + FileUtil.deleteDir(src); + FileUtil.deleteDir(dest); + } + } + + /** The result tar comes back from a container that ran customer script code. */ + @org.junit.Test + public void rejectsPathTraversal() throws IOException + { + File dest = FileUtil.createTempDirectory("arch-traversal").toFile(); + File tar = File.createTempFile("evil", ".tar"); + try + { + try (OutputStream fos = new FileOutputStream(tar); + TarArchiveOutputStream out = new TarArchiveOutputStream(fos)) + { + TarArchiveEntry entry = new TarArchiveEntry("work/../../escaped.txt"); + byte[] payload = "pwned".getBytes(); + entry.setSize(payload.length); + out.putArchiveEntry(entry); + out.write(payload); + out.closeArchiveEntry(); + out.finish(); + } + + try (InputStream in = new FileInputStream(tar)) + { + extract(in, dest); + fail("Expected traversal entry to be rejected"); + } + catch (IOException expected) + { + assertTrue(expected.getMessage().contains("outside the destination")); + } + } + finally + { + FileUtil.deleteDir(dest); + } + } + } +} diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 80282aae94b..37827e4fee2 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -1497,6 +1497,7 @@ public TabDisplayMode getTabDisplayMode() public @NotNull Set> getUnitTests() { return Set.of( + org.labkey.api.util.DirectoryArchive.TestCase.class, AdminController.FileRootPermissionTestCase.class, ApiJsonWriter.TestCase.class, ClassLoaderTestCase.class, diff --git a/core/src/org/labkey/core/reports/ExternalScriptEngineDefinitionImpl.java b/core/src/org/labkey/core/reports/ExternalScriptEngineDefinitionImpl.java index 4556fca4b6d..8ec6e3631ba 100644 --- a/core/src/org/labkey/core/reports/ExternalScriptEngineDefinitionImpl.java +++ b/core/src/org/labkey/core/reports/ExternalScriptEngineDefinitionImpl.java @@ -80,6 +80,7 @@ public class ExternalScriptEngineDefinitionImpl extends Entity implements Extern private boolean _external; private boolean _remote; private boolean _docker; + private boolean _remoteRunner; private boolean _pandocEnabled; private boolean _default; private boolean _sandboxed; @@ -169,6 +170,7 @@ public void updateConfiguration() addIfNotNull(json, "external", isExternal()); addIfNotNull(json, "remote", isRemote()); addIfNotNull(json, "docker", isDocker()); + addIfNotNull(json, "remoteRunner", isRemoteRunner()); addIfNotNull(json, "pandocEnabled", isPandocEnabled()); addIfNotNull(json, "fileExchange", getFileExchange()); addIfNotNull(json, "pathMap", _pathMap); @@ -234,6 +236,8 @@ public void setConfiguration(String configuration, boolean decrypt) throws IOExc setRemote(json.getBoolean("remote")); if (json.has("docker")) setDocker(json.getBoolean("docker")); + if (json.has("remoteRunner")) + setRemoteRunner(json.getBoolean("remoteRunner")); if (json.has("pandocEnabled")) setPandocEnabled(json.getBoolean("pandocEnabled")); if (json.has("fileExchange")) @@ -507,6 +511,28 @@ public void setDocker(boolean docker) _docker = docker; } + + @Override + + public boolean isRemoteRunner() + + { + + return _remoteRunner; + + } + + + @Override + + public void setRemoteRunner(boolean remoteRunner) + + { + + _remoteRunner = remoteRunner; + + } + @Override public boolean isPandocEnabled() { diff --git a/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java b/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java index aace2da21d4..7496bbb96b2 100644 --- a/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java +++ b/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java @@ -48,7 +48,9 @@ import org.labkey.api.reports.ExternalScriptEngineFactory; import org.labkey.api.reports.LabKeyScriptEngineManager; import org.labkey.api.reports.report.python.PythonScriptEngine; +import org.labkey.api.remoterunner.RemoteRunnerService; import org.labkey.api.reports.report.r.RDockerScriptEngineFactory; +import org.labkey.api.reports.report.r.RRemoteScriptEngineFactory; import org.labkey.api.reports.report.r.RScriptEngineFactory; import org.labkey.api.reports.report.r.RemoteRNotEnabledException; import org.labkey.api.reports.report.r.RserveScriptEngineFactory; @@ -330,6 +332,20 @@ public boolean isSandboxed() { if (def.isDocker()) return new RDockerScriptEngineFactory(def).getScriptEngine(); + else if (def.isRemoteRunner()) + { + RemoteRunnerService runner = RemoteRunnerService.get(); + if (null == runner || !runner.isEnabled()) + { + // Returning null here would silently fall back to running on the appserver, which is the thing + // this engine exists to avoid. + IllegalStateException ex = new IllegalStateException(String.format( + "R engine [%1$s] targets the remote script runner, but no runner is configured on this server.", def.getName())); + LOG.error(ex.getMessage()); + throw ex; + } + return new RRemoteScriptEngineFactory(def).getScriptEngine(); + } else if (def.isRemote()) { if (PremiumService.get().isRemoteREnabled()) From 7e20327de89b72aa19f20229e23570b72de51d53 Mon Sep 17 00:00:00 2001 From: Will Mooreston Date: Tue, 1 Sep 2026 21:33:54 -0700 Subject: [PATCH 2/9] Kanban 1743: let DirectoryArchive create bare entry names An empty prefix now yields unprefixed entries rather than ones beginning with a slash, so the archive can be extracted into any directory. --- api/src/org/labkey/api/util/DirectoryArchive.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/util/DirectoryArchive.java b/api/src/org/labkey/api/util/DirectoryArchive.java index dbee116dd89..2be57235ce9 100644 --- a/api/src/org/labkey/api/util/DirectoryArchive.java +++ b/api/src/org/labkey/api/util/DirectoryArchive.java @@ -52,7 +52,9 @@ private static void addDirectory(TarArchiveOutputStream tar, File dir, @Nullable for (File child : children) { - String entryName = prefix + "/" + child.getName(); + // An empty prefix means entries are named bare, so extracting anywhere reproduces the directory. Joining + // unconditionally would name them "/script.r", which extraction rejects as an absolute path. + String entryName = prefix.isEmpty() ? child.getName() : prefix + "/" + child.getName(); if (child.isDirectory()) { addDirectory(tar, child, filter, entryName); From fee8237b12c2c2b432c20162ab3e8e48acc5b863 Mon Sep 17 00:00:00 2001 From: Will Mooreston Date: Tue, 1 Sep 2026 22:11:21 -0700 Subject: [PATCH 3/9] Kanban 1743: extract tars flat instead of stripping a component extract() dropped every entry whose name had no slash, silently discarding all remote results. Removes the parentName parameter as well, so create and extract cannot disagree about prefixing again. --- .../org/labkey/api/util/DirectoryArchive.java | 102 +++++++++++++----- 1 file changed, 78 insertions(+), 24 deletions(-) diff --git a/api/src/org/labkey/api/util/DirectoryArchive.java b/api/src/org/labkey/api/util/DirectoryArchive.java index 2be57235ce9..cbebaf88b4c 100644 --- a/api/src/org/labkey/api/util/DirectoryArchive.java +++ b/api/src/org/labkey/api/util/DirectoryArchive.java @@ -31,15 +31,15 @@ private DirectoryArchive() { } - /** Tar {@code directory} into {@code target}. Entries are named relative to {@code parentName}. */ - public static void create(File directory, @Nullable FileFilter filter, String parentName, File target) throws IOException + /** Tar {@code directory} into {@code target}, with entries named relative to it. */ + public static void create(File directory, @Nullable FileFilter filter, File target) throws IOException { try (OutputStream fos = new FileOutputStream(target); TarArchiveOutputStream tar = new TarArchiveOutputStream(fos)) { // Long working-directory paths are routine, and the default format silently truncates past 100 chars. tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - addDirectory(tar, directory, filter, trimTrailingSlash(parentName)); + addDirectory(tar, directory, filter, ""); tar.finish(); } } @@ -52,8 +52,8 @@ private static void addDirectory(TarArchiveOutputStream tar, File dir, @Nullable for (File child : children) { - // An empty prefix means entries are named bare, so extracting anywhere reproduces the directory. Joining - // unconditionally would name them "/script.r", which extraction rejects as an absolute path. + // Top level is unprefixed; joining unconditionally would name it "/script.r", which extract rejects as an + // absolute path. The prefix only accumulates as recursion descends into subdirectories. String entryName = prefix.isEmpty() ? child.getName() : prefix + "/" + child.getName(); if (child.isDirectory()) { @@ -74,11 +74,10 @@ private static void addDirectory(TarArchiveOutputStream tar, File dir, @Nullable } /** - * Untar {@code in} beneath {@code destination}, stripping the leading path component so a tar made with - * {@code parentName} unpacks flat into the destination. + * Untar {@code in} beneath {@code destination}, reproducing the archived directory. * - * Entries resolving outside the destination are rejected. The tar arrives from a remote runner that executed - * customer script code, so its entry names are untrusted input. + * Entries resolving outside the destination are rejected, absolute names included. The tar arrives from a remote + * runner that executed customer script code, so its entry names are untrusted input. */ public static void extract(InputStream in, File destination) throws IOException { @@ -89,7 +88,7 @@ public static void extract(InputStream in, File destination) throws IOException TarArchiveEntry entry; while ((entry = tar.getNextEntry()) != null) { - String name = stripLeadingComponent(entry.getName()); + String name = entry.getName(); if (name.isEmpty()) continue; @@ -113,18 +112,6 @@ public static void extract(InputStream in, File destination) throws IOException } } - private static String stripLeadingComponent(String entryName) - { - String name = entryName.startsWith("/") ? entryName.substring(1) : entryName; - int slash = name.indexOf('/'); - return slash < 0 ? "" : name.substring(slash + 1); - } - - private static String trimTrailingSlash(String s) - { - return s.endsWith("/") ? s.substring(0, s.length() - 1) : s; - } - public static class TestCase extends org.junit.Assert { @org.junit.Test @@ -141,7 +128,7 @@ public void roundTripPreservesContentAndBinary() throws IOException Files.write(new File(nested, "plot.png").toPath(), binary); File tar = File.createTempFile("arch", ".tar"); - create(src, null, "/work", tar); + create(src, null, tar); try (InputStream in = new FileInputStream(tar)) { extract(in, dest); @@ -168,7 +155,7 @@ public void filterExcludesUnwantedFiles() throws IOException Files.writeString(new File(src, "drop.tmp").toPath(), "drop"); File tar = File.createTempFile("arch", ".tar"); - create(src, f -> f.getName().endsWith(".R"), "/work", tar); + create(src, f -> f.getName().endsWith(".R"), tar); try (InputStream in = new FileInputStream(tar)) { extract(in, dest); @@ -184,6 +171,73 @@ public void filterExcludesUnwantedFiles() throws IOException } } + /** The runner returns a flat tar. Skipping those entries loses every result without reporting anything. */ + @org.junit.Test + public void extractsFlatEntriesFromAForeignTar() throws IOException + { + File dest = FileUtil.createTempDirectory("arch-flat").toFile(); + File tar = File.createTempFile("flat", ".tar"); + try + { + writeSingleEntryTar(tar, "script.r.Rout", "console output"); + + try (InputStream in = new FileInputStream(tar)) + { + extract(in, dest); + } + + assertEquals("console output", Files.readString(new File(dest, "script.r.Rout").toPath())); + } + finally + { + FileUtil.deleteDir(dest); + } + } + + /** An absolute entry would otherwise escape the destination entirely. */ + @org.junit.Test + public void rejectsAbsoluteEntry() throws IOException + { + File dest = FileUtil.createTempDirectory("arch-absolute").toFile(); + File tar = File.createTempFile("absolute", ".tar"); + try + { + writeSingleEntryTar(tar, "/etc/passwd", "pwned"); + + try (InputStream in = new FileInputStream(tar)) + { + extract(in, dest); + fail("Expected absolute entry to be rejected"); + } + catch (IOException expected) + { + assertTrue(expected.getMessage().contains("outside the destination")); + } + } + finally + { + FileUtil.deleteDir(dest); + } + } + + private static void writeSingleEntryTar(File tar, String entryName, String content) throws IOException + { + try (OutputStream fos = new FileOutputStream(tar); + TarArchiveOutputStream out = new TarArchiveOutputStream(fos)) + { + out.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + // preserveAbsolutePath, or the constructor strips the leading slash and the absolute case cannot be + // written at all. Python's tarfile preserves it, so the runner can produce one. + TarArchiveEntry entry = new TarArchiveEntry(entryName, true); + byte[] payload = content.getBytes(); + entry.setSize(payload.length); + out.putArchiveEntry(entry); + out.write(payload); + out.closeArchiveEntry(); + out.finish(); + } + } + /** The result tar comes back from a container that ran customer script code. */ @org.junit.Test public void rejectsPathTraversal() throws IOException From ddeb137ae2a175b529c13ca2421a40b7d42daf3c Mon Sep 17 00:00:00 2001 From: Will Mooreston Date: Tue, 1 Sep 2026 22:11:38 -0700 Subject: [PATCH 4/9] Kanban 1743: capture R package usage by relative path The inherited epilog names the appserver's working directory, which does not exist in the container, so nothing was ever recorded. --- .../reports/report/r/RRemoteScriptEngine.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java index 989b78410af..97afaec034c 100644 --- a/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java +++ b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java @@ -103,6 +103,23 @@ protected Object eval(FileLike scriptFile, ScriptContext context) throws ScriptE return scriptOut; } + /** + * The base class names this file by the appserver's working directory, which does not exist in the container, so + * the write silently fails and no packages are ever recorded. The runner runs the script from the unpacked job + * directory, so a bare name lands beside the script and returns in the result tar. + */ + @Override + protected @Nullable String getPackageCaptureEpilog(ScriptContext context) + { + if (getKnitrFormat(context) != RReportDescriptor.KnitrFormat.None) + return null; + + return """ + # --- LabKey R package usage capture --- + tryCatch(writeLines(sort(loadedNamespaces()), "%s"), error = function(e) invisible(NULL)) + """.formatted(PACKAGES_FILE); + } + private static FileFilter inputFiles() { return pathname -> From f14cf353e4b9cedff524ff15c1392519a772c6ed Mon Sep 17 00:00:00 2001 From: Will Mooreston Date: Tue, 1 Sep 2026 22:22:30 -0700 Subject: [PATCH 5/9] Kanban 1743: correct the remoteWorkingDir javadoc It no longer affects tar entry names, only path mapping. --- api/src/org/labkey/api/remoterunner/RemoteRunnerService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java b/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java index c7b4bd7188c..7139ee5d539 100644 --- a/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java +++ b/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java @@ -38,7 +38,7 @@ static void setInstance(RemoteRunnerService impl) * * @param scriptFile the script to run, already written into the working directory * @param localWorkingDir working directory on this server - * @param remoteWorkingDir path the runner will see, used for tar entry names and path mapping + * @param remoteWorkingDir path the runner will see, used for path mapping * @param inputFiles which files in the working directory to send */ void executeR(FileLike scriptFile, String localWorkingDir, String remoteWorkingDir, @Nullable FileFilter inputFiles) From b0d3d15983a57a49948ba3ca9f9989ca37d84572 Mon Sep 17 00:00:00 2001 From: Will Mooreston Date: Tue, 1 Sep 2026 23:00:33 -0700 Subject: [PATCH 6/9] Kanban 1743: rename executeR to execute The runner selects an interpreter from the script extension, so nothing about the service contract is R-specific. --- api/src/org/labkey/api/remoterunner/RemoteRunnerService.java | 5 ++++- .../org/labkey/api/reports/report/r/RRemoteScriptEngine.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java b/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java index 7139ee5d539..758ce598110 100644 --- a/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java +++ b/api/src/org/labkey/api/remoterunner/RemoteRunnerService.java @@ -36,11 +36,14 @@ static void setInstance(RemoteRunnerService impl) * Tar {@code localWorkingDir}, run {@code scriptFile} against it in the remote runner, and unpack the result back * over {@code localWorkingDir}. * + * The runner picks an interpreter from the script's extension, so this is not R-specific; an extension it does not + * recognize is an error rather than a default. + * * @param scriptFile the script to run, already written into the working directory * @param localWorkingDir working directory on this server * @param remoteWorkingDir path the runner will see, used for path mapping * @param inputFiles which files in the working directory to send */ - void executeR(FileLike scriptFile, String localWorkingDir, String remoteWorkingDir, @Nullable FileFilter inputFiles) + void execute(FileLike scriptFile, String localWorkingDir, String remoteWorkingDir, @Nullable FileFilter inputFiles) throws IOException; } diff --git a/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java index 97afaec034c..81ee9bf0e67 100644 --- a/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java +++ b/api/src/org/labkey/api/reports/report/r/RRemoteScriptEngine.java @@ -87,7 +87,7 @@ protected Object eval(FileLike scriptFile, ScriptContext context) throws ScriptE StringBuffer output = new StringBuffer(); try (CustomTiming t = MiniProfiler.custom("remoteRunner", "execute r in remote runner")) { - _service.executeR(scriptFile, getRWorkingDir(context), REMOTE_WORKING_DIR, inputFiles()); + _service.execute(scriptFile, getRWorkingDir(context), REMOTE_WORKING_DIR, inputFiles()); appendConsoleOutput(context, output); } catch (Exception e) From 4f2d8b6994e2526bb43a47c7c7c32f768bc81719 Mon Sep 17 00:00:00 2001 From: Will Mooreston Date: Tue, 1 Sep 2026 23:00:40 -0700 Subject: [PATCH 7/9] Kanban 1743: offer a remote runner R engine on the scripting page Follows the docker engine: the runner's endpoint and bucket are site-wide, so the engine form shows their status and links to the settings page rather than repeating them per engine. --- .../core/view/configReportsAndScripts.jsp | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/core/src/org/labkey/core/view/configReportsAndScripts.jsp b/core/src/org/labkey/core/view/configReportsAndScripts.jsp index 73dc792ff51..1f773b900ac 100644 --- a/core/src/org/labkey/core/view/configReportsAndScripts.jsp +++ b/core/src/org/labkey/core/view/configReportsAndScripts.jsp @@ -19,6 +19,7 @@ <%@ page import="org.labkey.api.docker.DockerService"%> <%@ page import="org.labkey.api.files.FileContentService"%> <%@ page import="org.labkey.api.premium.PremiumService" %> +<%@ page import="org.labkey.api.remoterunner.RemoteRunnerService" %> <%@ page import="org.labkey.api.reports.ExternalScriptEngine" %> <%@ page import="org.labkey.api.reports.ExternalScriptEngineDefinition" %> <%@ page import="org.labkey.api.reports.report.ExternalScriptEngineReport" %> @@ -51,6 +52,10 @@ } boolean hasAdminOpsPerms = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); boolean baseServerUrlSet = !AppProps.getInstance().getBaseServerUrl().contains("localhost"); + // Offered whenever the module supplying the runner is deployed; the form reports whether it is actually configured. + RemoteRunnerService remoteRunnerService = RemoteRunnerService.get(); + boolean isRemoteRunnerAvailable = null != remoteRunnerService; + boolean isRemoteRunnerConfigured = null != remoteRunnerService && remoteRunnerService.isEnabled(); %>