diff --git a/Jenkinsfile b/Jenkinsfile index ba067316..927775bd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,8 +51,8 @@ pipeline { } } stage('GPU') { - agent { label 'ephemeral-linux-gpu' } - stages { + agent { label 'ephemeral-linux-gpu-t4x2' } + stages { stage('Build GPU Image') { options { timeout(time: 4324, unit: 'MINUTES') @@ -135,23 +135,6 @@ pipeline { } } } - stage('Test on P100') { - agent { label 'ephemeral-linux-gpu' } - options { - timeout(time: 40, unit: 'MINUTES') - } - steps { - retry(2) { - sh '''#!/bin/bash - set -exo pipefail - - date - docker pull gcr.io/kaggle-private-byod/python:${PRETEST_TAG} - ./test --gpu --image gcr.io/kaggle-private-byod/python:${PRETEST_TAG} - ''' - } - } - } stage('Test on T4x2') { agent { label 'ephemeral-linux-gpu-t4x2' } options { diff --git a/tests/common.py b/tests/common.py index 469033dd..5ab1c6ca 100644 --- a/tests/common.py +++ b/tests/common.py @@ -15,6 +15,4 @@ def isGPU(): return os.path.isfile('/proc/driver/nvidia/version') gpu_test = unittest.skipIf(not isGPU(), 'Not running GPU tests') -# b/342143152 P100s are slowly being unsupported in new release of popular ml tools such as RAPIDS. -p100_exempt = unittest.skipIf(getAcceleratorName() == "Tesla P100-PCIE-16GB", 'Not running p100 exempt tests') tpu_test = unittest.skipIf(len(os.environ.get('ISTPUVM', '')) == 0, 'Not running TPU tests') diff --git a/tests/test_cudf.py b/tests/test_cudf.py index cae69a6c..f8df3fa8 100644 --- a/tests/test_cudf.py +++ b/tests/test_cudf.py @@ -1,11 +1,10 @@ import unittest -from common import gpu_test, p100_exempt +from common import gpu_test class TestCudf(unittest.TestCase): @gpu_test - @p100_exempt # b/342143152: cuDL(>=24.4v) is inompatible with p100 GPUs. def test_cudf_dataframe_operations(self): import cudf diff --git a/tests/test_cuml.py b/tests/test_cuml.py index bbb7f0c6..42ea7b46 100644 --- a/tests/test_cuml.py +++ b/tests/test_cuml.py @@ -1,11 +1,10 @@ import unittest -from common import gpu_test, p100_exempt +from common import gpu_test class TestCuml(unittest.TestCase): @gpu_test - @p100_exempt # b/342143152: cuML(>=24.4v) is inompatible with p100 GPUs. def test_pca_fit_transform(self): import unittest import numpy as np diff --git a/tests/test_fastai.py b/tests/test_fastai.py index 497af048..33a436a5 100644 --- a/tests/test_fastai.py +++ b/tests/test_fastai.py @@ -3,8 +3,6 @@ import fastai from fastai.tabular.all import * -from common import p100_exempt - class TestFastAI(unittest.TestCase): # Basic import @@ -24,7 +22,6 @@ def test_torch_tensor(self): self.assertTrue(torch.all(a == b)) - @p100_exempt def test_tabular(self): dls = TabularDataLoaders.from_csv( "/input/tests/data/train.csv", diff --git a/tests/test_flax.py b/tests/test_flax.py index 5365f7c1..f17e3949 100644 --- a/tests/test_flax.py +++ b/tests/test_flax.py @@ -8,8 +8,6 @@ from flax import linen as nn from flax.training import train_state -from common import p100_exempt - class TestFlax(unittest.TestCase): @@ -19,12 +17,6 @@ def test_pooling(self): y = nn.pooling.pool(x, 1., mul_reduce, (2, 2), (1, 1), 'VALID') np.testing.assert_allclose(y, np.full((1, 2, 2, 1), 2. ** 4)) - # cuDNN 9.19 (pulled in by torch 2.11) dropped the Pascal (sm_60) kernels from - # libcudnn_ops/cnn/adv and ships PTX for sm_121 only, so nothing can be JIT'd - # down to sm_60 either. Every cuDNN convolution engine fails on P100 with - # CUDNN_STATUS_EXECUTION_FAILED. Not fixable here: cuDNN 9.19 is a hard - # requirement of torch 2.11, which comes from the Colab base image. - @p100_exempt def test_cnn(self): class CNN(nn.Module): @nn.compact diff --git a/tests/test_keras.py b/tests/test_keras.py index 0090178b..5dc4610d 100644 --- a/tests/test_keras.py +++ b/tests/test_keras.py @@ -7,15 +7,7 @@ import keras -from common import p100_exempt - class TestKeras(unittest.TestCase): - # cuDNN 9.19 (pulled in by torch 2.11) dropped the Pascal (sm_60) kernels from - # libcudnn_ops/cnn/adv and ships PTX for sm_121 only, so nothing can be JIT'd - # down to sm_60 either. Every cuDNN convolution engine fails on P100 with - # CUDNN_STATUS_EXECUTION_FAILED. Not fixable here: cuDNN 9.19 is a hard - # requirement of torch 2.11, which comes from the Colab base image. - @p100_exempt def test_train(self): path = '/input/tests/data/mnist.npz' with np.load(path) as f: diff --git a/tests/test_keras_cv.py b/tests/test_keras_cv.py index be8b1a27..fb5d912c 100644 --- a/tests/test_keras_cv.py +++ b/tests/test_keras_cv.py @@ -4,16 +4,9 @@ import keras import numpy as np -from common import p100_exempt from utils.kagglehub import create_test_kagglehub_server class TestKerasCV(unittest.TestCase): - # cuDNN 9.19 (pulled in by torch 2.11) dropped the Pascal (sm_60) kernels from - # libcudnn_ops/cnn/adv and ships PTX for sm_121 only, so nothing can be JIT'd - # down to sm_60 either. Every cuDNN convolution engine fails on P100 with - # CUDNN_STATUS_EXECUTION_FAILED. Not fixable here: cuDNN 9.19 is a hard - # requirement of torch 2.11, which comes from the Colab base image. - @p100_exempt def test_inference(self): with create_test_kagglehub_server(): classifier = keras_cv.models.ImageClassifier.from_preset( diff --git a/tests/test_pytorch.py b/tests/test_pytorch.py index 31b008d5..befa6f7e 100644 --- a/tests/test_pytorch.py +++ b/tests/test_pytorch.py @@ -4,7 +4,7 @@ import torch.nn as tnn import torch.autograd as autograd -from common import gpu_test, p100_exempt +from common import gpu_test class TestPyTorch(unittest.TestCase): @@ -16,7 +16,6 @@ def test_nn(self): linear_torch(data_torch) @gpu_test - @p100_exempt def test_linalg(self): A = torch.randn(3, 3).t().to('cuda') B = torch.randn(3).t().to('cuda') @@ -25,7 +24,6 @@ def test_linalg(self): self.assertEqual(3, result.shape[0]) @gpu_test - @p100_exempt def test_gpu_computation(self): cuda = torch.device('cuda') a = torch.tensor([1., 2.], device=cuda) @@ -35,7 +33,6 @@ def test_gpu_computation(self): self.assertEqual(torch.tensor([3.], device=cuda), result) @gpu_test - @p100_exempt def test_cuda_nn(self): # These throw if cuda is misconfigured tnn.GRUCell(10,10).cuda() diff --git a/tests/test_pytorch_lightning.py b/tests/test_pytorch_lightning.py index feead123..24992e5f 100644 --- a/tests/test_pytorch_lightning.py +++ b/tests/test_pytorch_lightning.py @@ -5,8 +5,6 @@ import torch.nn.functional as F from torch.utils.data import DataLoader, TensorDataset -from common import p100_exempt - class LitDataModule(pl.LightningDataModule): @@ -61,7 +59,6 @@ class TestPytorchLightning(unittest.TestCase): def test_version(self): self.assertIsNotNone(pl.__version__) - @p100_exempt def test_mnist(self): dm = LitDataModule() model = LitClassifier() diff --git a/tests/test_tensorflow.py b/tests/test_tensorflow.py index d8fa1e79..8b415e25 100644 --- a/tests/test_tensorflow.py +++ b/tests/test_tensorflow.py @@ -4,7 +4,7 @@ import numpy as np import tensorflow as tf -from common import gpu_test, p100_exempt +from common import gpu_test class TestTensorflow(unittest.TestCase): @@ -46,12 +46,6 @@ def test_tf_keras(self): tf.keras.utils.plot_model(model, to_file="tf_plot_model.png") self.assertTrue(os.path.isfile("tf_plot_model.png")) - # cuDNN 9.19 (pulled in by torch 2.11) dropped the Pascal (sm_60) kernels from - # libcudnn_ops/cnn/adv and ships PTX for sm_121 only, so nothing can be JIT'd - # down to sm_60 either. CudnnRNNV3 fails on P100 with - # CUDNN_STATUS_EXECUTION_FAILED. Not fixable here: cuDNN 9.19 is a hard - # requirement of torch 2.11, which comes from the Colab base image. - @p100_exempt def test_lstm(self): x_train = np.random.random((100, 28, 28)) y_train = np.random.randint(10, size=(100, 1))