Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit d6ad82a

Browse filesBrowse files
sunnylovestiramisusunnylovestiramisu
committed
Add Stress Tests for VolumeAttributesClass
1 parent cec0492 commit d6ad82a
Copy full SHA for d6ad82a

File tree

3 files changed

+269
-0
lines changed
Filter options

3 files changed

+269
-0
lines changed

‎test/e2e/storage/drivers/csi.go

Copy file name to clipboardExpand all lines: test/e2e/storage/drivers/csi.go
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@ func initHostPathCSIDriver(name string, capabilities map[storageframework.Capabi
120120
NumPods: 10,
121121
NumSnapshots: 10,
122122
},
123+
VolumeModifyStressTestOptions: &storageframework.VolumeModifyStressTestOptions{
124+
NumPods: 10,
125+
},
123126
PerformanceTestOptions: &storageframework.PerformanceTestOptions{
124127
ProvisioningOptions: &storageframework.PerformanceTestProvisioningOptions{
125128
VolumeSize: "1Mi",

‎test/e2e/storage/framework/testdriver.go

Copy file name to clipboardExpand all lines: test/e2e/storage/framework/testdriver.go
+9Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,8 @@ type DriverInfo struct {
264264
StressTestOptions *StressTestOptions
265265
// [Optional] Scale parameters for volume snapshot stress tests.
266266
VolumeSnapshotStressTestOptions *VolumeSnapshotStressTestOptions
267+
// [Optional] Scale parameters for volume modify stress tests.
268+
VolumeModifyStressTestOptions *VolumeModifyStressTestOptions
267269
// [Optional] Parameters for performance tests
268270
PerformanceTestOptions *PerformanceTestOptions
269271
}
@@ -286,6 +288,13 @@ type VolumeSnapshotStressTestOptions struct {
286288
NumSnapshots int
287289
}
288290

291+
// VolumeModifyStressTestOptions contains parameters used for volume modify stress tests.
292+
type VolumeModifyStressTestOptions struct {
293+
// Number of pods to create in the test. This may also create
294+
// up to 1 volume with volumeAttributesClass per pod.
295+
NumPods int
296+
}
297+
289298
// Metrics to evaluate performance of an operation
290299
// TODO: Add metrics like median, mode, standard deviation, percentile
291300
type Metrics struct {

‎test/e2e/storage/testsuites/volume_modify_stress.go

Copy file name to clipboard
+257Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package testsuites
18+
19+
import (
20+
"context"
21+
"sync"
22+
23+
"github.com/onsi/ginkgo/v2"
24+
v1 "k8s.io/api/core/v1"
25+
storagev1beta1 "k8s.io/api/storage/v1beta1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/util/errors"
28+
clientset "k8s.io/client-go/kubernetes"
29+
"k8s.io/kubernetes/pkg/features"
30+
e2efeature "k8s.io/kubernetes/test/e2e/feature"
31+
"k8s.io/kubernetes/test/e2e/framework"
32+
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
33+
e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
34+
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
35+
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
36+
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
37+
admissionapi "k8s.io/pod-security-admission/api"
38+
)
39+
40+
type volumeModifyStressTestSuite struct {
41+
tsInfo storageframework.TestSuiteInfo
42+
}
43+
44+
type volumeModifyStressTest struct {
45+
config *storageframework.PerTestConfig
46+
47+
migrationCheck *migrationOpCheck
48+
49+
vac *storagev1beta1.VolumeAttributesClass
50+
51+
volumes []*storageframework.VolumeResource
52+
pods []*v1.Pod
53+
// stop and wait for any async routines
54+
wg sync.WaitGroup
55+
56+
testOptions storageframework.StressTestOptions
57+
}
58+
59+
var _ storageframework.TestSuite = &volumeModifyStressTestSuite{}
60+
61+
// InitCustomVolumeModifyStressTestSuite returns volumeModifyStressTestSuite that implements TestSuite interface
62+
// using custom test patterns
63+
func InitCustomVolumeModifyStressTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
64+
return &volumeModifyStressTestSuite{
65+
tsInfo: storageframework.TestSuiteInfo{
66+
Name: "volume-modify-stress",
67+
TestPatterns: patterns,
68+
SupportedSizeRange: e2evolume.SizeRange{
69+
Min: "1Gi",
70+
},
71+
TestTags: []interface{}{e2efeature.VolumeAttributesClass, framework.WithFeatureGate(features.VolumeAttributesClass)},
72+
},
73+
}
74+
}
75+
76+
// InitVolumeModifyStressTestSuite returns volumeModifyStressTestSuite that implements TestSuite interface
77+
// using testsuite default patterns
78+
func InitVolumeModifyStressTestSuite() storageframework.TestSuite {
79+
patterns := []storageframework.TestPattern{
80+
storageframework.DefaultFsDynamicPV,
81+
storageframework.BlockVolModeDynamicPV,
82+
storageframework.NtfsDynamicPV,
83+
}
84+
return InitCustomVolumeModifyStressTestSuite(patterns)
85+
}
86+
87+
func (v *volumeModifyStressTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
88+
return v.tsInfo
89+
}
90+
91+
func (v *volumeModifyStressTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
92+
driverInfo := driver.GetDriverInfo()
93+
if driverInfo.VolumeModifyStressTestOptions == nil {
94+
e2eskipper.Skipf("Driver %s doesn't specify volume modify stress test options -- skipping", driverInfo.Name)
95+
}
96+
if driverInfo.VolumeModifyStressTestOptions.NumPods <= 0 {
97+
framework.Failf("NumPods in modify volume stress test options must be a positive integer, received: %d", driverInfo.VolumeModifyStressTestOptions.NumPods)
98+
}
99+
_, ok := driver.(storageframework.VolumeAttributesClassTestDriver)
100+
if !ok {
101+
e2eskipper.Skipf("Driver %q does not support VolumeAttributesClass tests - skipping", driver.GetDriverInfo().Name)
102+
}
103+
// Skip block storage tests if the driver we are testing against does not support block volumes
104+
// TODO: This should be made generic so that it doesn't have to be re-written for every test that uses the BlockVolModeDynamicPV testcase
105+
if !driverInfo.Capabilities[storageframework.CapBlock] && pattern.VolMode == v1.PersistentVolumeBlock {
106+
e2eskipper.Skipf("Driver %q does not support block volume mode - skipping", driver.GetDriverInfo().Name)
107+
}
108+
}
109+
110+
func (v *volumeModifyStressTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
111+
112+
var (
113+
dInfo = driver.GetDriverInfo()
114+
cs clientset.Interface
115+
l *volumeModifyStressTest
116+
)
117+
118+
// Beware that it also registers an AfterEach which renders f unusable. Any code using
119+
// f must run inside an It or Context callback.
120+
f := framework.NewFrameworkWithCustomTimeouts("volume-modify-stress", storageframework.GetDriverTimeouts(driver))
121+
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
122+
123+
init := func(ctx context.Context) {
124+
cs = f.ClientSet
125+
l = &volumeModifyStressTest{}
126+
127+
// Now do the more expensive test initialization.
128+
l.config = driver.PrepareTest(ctx, f)
129+
vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver)
130+
l.volumes = []*storageframework.VolumeResource{}
131+
l.pods = []*v1.Pod{}
132+
l.vac = vacDriver.GetVolumeAttributesClass(ctx, l.config)
133+
134+
if l.vac == nil {
135+
e2eskipper.Skipf("Driver %q returned nil VolumeAttributesClass - skipping", dInfo.Name)
136+
}
137+
138+
ginkgo.By("Creating VolumeAttributesClass")
139+
_, err := f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Create(ctx, l.vac, metav1.CreateOptions{})
140+
framework.ExpectNoError(err, "While creating VolumeAttributesClass")
141+
}
142+
143+
createPodsAndVolumes := func(ctx context.Context) {
144+
for i := 0; i < l.testOptions.NumPods; i++ {
145+
framework.Logf("Creating resources for pod %v/%v", i, l.testOptions.NumPods-1)
146+
testVolumeSizeRange := v.GetTestSuiteInfo().SupportedSizeRange
147+
r := storageframework.CreateVolumeResourceWithVAC(ctx, driver, l.config, pattern, testVolumeSizeRange, &l.vac.Name)
148+
l.volumes = append(l.volumes, r)
149+
podConfig := e2epod.Config{
150+
NS: f.Namespace.Name,
151+
PVCs: []*v1.PersistentVolumeClaim{r.Pvc},
152+
SeLinuxLabel: e2epv.SELinuxLabel,
153+
}
154+
pod, err := e2epod.MakeSecPod(&podConfig)
155+
framework.ExpectNoError(err)
156+
157+
l.pods = append(l.pods, pod)
158+
}
159+
}
160+
161+
cleanup := func(ctx context.Context) {
162+
framework.Logf("Stopping and waiting for all test routines to finish")
163+
l.wg.Wait()
164+
165+
var (
166+
errs []error
167+
mu sync.Mutex
168+
wg sync.WaitGroup
169+
)
170+
171+
if l.vac != nil {
172+
ginkgo.By("Deleting VAC")
173+
CleanupVAC(ctx, l.vac, f.ClientSet, vacCleanupWaitPeriod)
174+
l.vac = nil
175+
}
176+
177+
wg.Add(len(l.pods))
178+
for _, pod := range l.pods {
179+
go func(pod *v1.Pod) {
180+
defer ginkgo.GinkgoRecover()
181+
defer wg.Done()
182+
183+
framework.Logf("Deleting pod %v", pod.Name)
184+
err := e2epod.DeletePodWithWait(ctx, cs, pod)
185+
mu.Lock()
186+
defer mu.Unlock()
187+
errs = append(errs, err)
188+
}(pod)
189+
}
190+
wg.Wait()
191+
192+
wg.Add(len(l.volumes))
193+
for _, volume := range l.volumes {
194+
go func(volume *storageframework.VolumeResource) {
195+
defer ginkgo.GinkgoRecover()
196+
defer wg.Done()
197+
198+
framework.Logf("Deleting volume %s", volume.Pvc.GetName())
199+
err := volume.CleanupResource(ctx)
200+
mu.Lock()
201+
defer mu.Unlock()
202+
errs = append(errs, err)
203+
}(volume)
204+
}
205+
wg.Wait()
206+
207+
framework.ExpectNoError(errors.NewAggregate(errs), "while cleaning up resource")
208+
l.migrationCheck.validateMigrationVolumeOpCounts(ctx)
209+
}
210+
211+
f.It("multiple pods should provision volumes with volumeAttributesClass", f.WithSlow(), f.WithSerial(), func(ctx context.Context) {
212+
init(ctx)
213+
ginkgo.DeferCleanup(cleanup)
214+
createPodsAndVolumes(ctx)
215+
// Restart pod repeatedly
216+
for i := 0; i < l.testOptions.NumPods; i++ {
217+
podIndex := i
218+
l.wg.Add(1)
219+
go func() {
220+
defer ginkgo.GinkgoRecover()
221+
defer l.wg.Done()
222+
for j := 0; j < l.testOptions.NumRestarts; j++ {
223+
select {
224+
case <-ctx.Done():
225+
// This looks like a in the
226+
// original test
227+
// (https://github.com/kubernetes/kubernetes/blob/21049c2a1234ae3eea57357ed4329ed567a2dab3/test/e2e/storage/testsuites/volume_stress.go#L212):
228+
// This early return will never
229+
// get reached even if some
230+
// other goroutine fails
231+
// because the context doesn't
232+
// get cancelled.
233+
return
234+
default:
235+
pod := l.pods[podIndex]
236+
framework.Logf("Pod-%v [%v], Iteration %v/%v", podIndex, pod.Name, j, l.testOptions.NumRestarts-1)
237+
_, err := cs.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
238+
if err != nil {
239+
framework.Failf("Failed to create pod-%v [%+v]. Error: %v", podIndex, pod, err)
240+
}
241+
242+
err = e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, pod.Name, pod.Namespace, f.Timeouts.PodStart)
243+
if err != nil {
244+
framework.Failf("Failed to wait for pod-%v [%+v] turn into running status. Error: %v", podIndex, pod, err)
245+
}
246+
247+
err = e2epod.DeletePodWithWait(ctx, f.ClientSet, pod)
248+
if err != nil {
249+
framework.Failf("Failed to delete pod-%v [%+v]. Error: %v", podIndex, pod, err)
250+
}
251+
}
252+
}
253+
}()
254+
}
255+
l.wg.Wait()
256+
})
257+
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.