Skip to content

Commit fd4540c

Browse files
rohit-arora-devrohit-arora-dev
authored
Revert publish info field from rawDevicePath to devicePath for raw devices
This change reverts publish info field from `rawDevicePath` to `devicePath` for raw devices. It identifies the missing publish paths in the tracking file for raw devices and also makes a better effort to recover devicePath field that may have been lost during pre-23.01 to 23.01/23.04 upgrades.
1 parent 8642bee commit fd4540c

12 files changed

Lines changed: 357 additions & 118 deletions

File tree

‎frontend/csi/node_helpers/kubernetes/plugin.go‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,15 @@ import (
2323
var osFs = afero.NewOsFs()
2424

2525
const (
26-
kubeDirEnvVar = "KUBELET_DIR"
27-
volumesFilesystemPath = "/volumes/kubernetes.io~csi/"
26+
kubeDirEnvVar = "KUBELET_DIR"
27+
volumesFilesystemPath = "/volumes/kubernetes.io~csi/"
28+
rawDevicePublishedPath = "/plugins/kubernetes.io/csi/volumeDevices/publish/"
2829
)
2930

3031
type helper struct {
3132
orchestrator core.Orchestrator
3233
podsPath string
34+
kubeConfigPath string
3335
publishedPaths map[string]map[string]struct{}
3436
enableForceDetach bool
3537
nodehelpers.VolumePublishManager // Embedded/extended interface
@@ -48,6 +50,7 @@ func NewHelper(orchestrator core.Orchestrator, kubeConfigPath string, enableForc
4850
h := &helper{
4951
orchestrator: orchestrator,
5052
podsPath: kubeConfigPath + "/pods",
53+
kubeConfigPath: kubeConfigPath,
5154
publishedPaths: make(map[string]map[string]struct{}),
5255
enableForceDetach: enableForceDetach,
5356
VolumePublishManager: csi.NewVolumePublishManager(config.VolumeTrackingInfoPath),
@@ -103,10 +106,16 @@ func (h *helper) reconcileVolumePublishInfo(ctx context.Context) error {
103106
}
104107

105108
if len(files) > 0 {
106-
publishedPaths, err := h.discoverPVCsToPublishedPaths(ctx)
109+
publishedPaths, err := h.discoverPVCsToPublishedPathsFilesystemVolumes(ctx)
107110
if err != nil {
108111
return fmt.Errorf("could not discover published paths: %v", err)
109112
}
113+
114+
err = h.discoverPVCsToPublishedPathsRawDevices(ctx, publishedPaths)
115+
if err != nil {
116+
return fmt.Errorf("could not discover published raw devices: %v", err)
117+
}
118+
110119
h.publishedPaths = publishedPaths
111120
}
112121

@@ -211,8 +220,8 @@ func (h *helper) RemovePublishedPath(ctx context.Context, volumeID, pathToRemove
211220
return nil
212221
}
213222

214-
// discoverPVCsToPublishedPaths builds a map of PVCs to the Pods they are mounted to and returns it.
215-
func (h *helper) discoverPVCsToPublishedPaths(ctx context.Context) (map[string]map[string]struct{}, error) {
223+
// discoverPVCsToPublishedPathsFilesystemVolumes builds a map of PVCs to the Pods they are mounted to and returns it.
224+
func (h *helper) discoverPVCsToPublishedPathsFilesystemVolumes(ctx context.Context) (map[string]map[string]struct{}, error) {
216225
// VolumeID -> PublishPaths
217226
mapping := make(map[string]map[string]struct{})
218227

@@ -250,3 +259,52 @@ func (h *helper) discoverPVCsToPublishedPaths(ctx context.Context) (map[string]m
250259
Logc(ctx).WithFields(LogFields{"publishedPaths": mapping}).Debug("Discovered PVC mount points.")
251260
return mapping, nil
252261
}
262+
263+
// discoverPVCsToPublishedPathsRawDevices builds a map of PVCs (
264+
// raw blocks) to the Pods they are mounted to and returns it.
265+
func (h *helper) discoverPVCsToPublishedPathsRawDevices(ctx context.Context, mapping map[string]map[string]struct{}) error {
266+
// VolumeID -> PublishPaths
267+
268+
if mapping == nil {
269+
mapping = make(map[string]map[string]struct{})
270+
}
271+
272+
Logc(ctx).Debug("Discovering PVC attachements...")
273+
publishedRawDevicePath := filepath.Join(h.kubeConfigPath, rawDevicePublishedPath)
274+
fields := LogFields{"publishedRawDevicePath": publishedRawDevicePath}
275+
276+
volumes, err := afero.ReadDir(osFs, publishedRawDevicePath)
277+
if err != nil && !os.IsNotExist(err) {
278+
Logc(ctx).WithFields(fields).Errorf("Error reading raw device directory; %v", err)
279+
return err
280+
}
281+
282+
for _, volume := range volumes {
283+
if !strings.Contains(volume.Name(), "pvc-") {
284+
continue
285+
}
286+
287+
if mapping[volume.Name()] == nil {
288+
mapping[volume.Name()] = make(map[string]struct{})
289+
}
290+
291+
volumePath := filepath.Join(publishedRawDevicePath, volume.Name())
292+
fields = LogFields{"volumePath": volumePath}
293+
podUUIDs, err := afero.ReadDir(osFs, volumePath)
294+
if err != nil && !os.IsNotExist(err) {
295+
Logc(ctx).WithFields(fields).Errorf("Error reading pods path; %v", err)
296+
return err
297+
}
298+
299+
for _, podUUID := range podUUIDs {
300+
pubPath := filepath.Join(volumePath, podUUID.Name())
301+
fields := LogFields{"volumeId": volume.Name(), "publishedPath": pubPath}
302+
Logc(ctx).WithFields(fields).Debug("Found published path for volume.")
303+
mapping[volume.Name()][pubPath] = struct{}{}
304+
}
305+
306+
}
307+
308+
Logc(ctx).WithFields(LogFields{"publishedPaths": mapping}).Debug("Discovered PVC raw device mount points.")
309+
return nil
310+
}

‎frontend/csi/node_helpers/kubernetes/plugin_test.go‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ func TestDiscoverPVCsToPublishedPaths(t *testing.T) {
358358
_ = osFs.MkdirAll(podUUIDPath, 0o777)
359359
_, _ = osFs.Create(podUUIDPath + volName)
360360

361-
result, err := h.discoverPVCsToPublishedPaths(context.Background())
361+
result, err := h.discoverPVCsToPublishedPathsFilesystemVolumes(context.Background())
362362
expectedPublishedPath := filepath.Join(podUUIDPath, volName, "mount")
363363
_, ok = result[volName][expectedPublishedPath]
364364
assert.NoError(t, err)
@@ -384,19 +384,92 @@ func TestDiscoverPVCsToPublishedPaths_ReadDirFails(t *testing.T) {
384384

385385
// invalid path
386386
h.podsPath = "/*"
387-
res, err := h.discoverPVCsToPublishedPaths(context.Background())
387+
res, err := h.discoverPVCsToPublishedPathsFilesystemVolumes(context.Background())
388388
assert.Error(t, err)
389389
assert.Nil(t, res, "expected nil map!")
390390

391391
// remove directory, then make a file with the same name
392392
_ = osFs.RemoveAll(podUUIDPath)
393393
_, _ = osFs.Create(podUUIDPath)
394394
h.podsPath = "/var/lib/kubelet/pods"
395-
res, err = h.discoverPVCsToPublishedPaths(context.Background())
395+
res, err = h.discoverPVCsToPublishedPathsFilesystemVolumes(context.Background())
396396
assert.Error(t, err)
397397
assert.Empty(t, res)
398398
}
399399

400+
func TestDiscoverPVCsToPublishedPathsRawDevices(t *testing.T) {
401+
mockCtrl := gomock.NewController(t)
402+
orchestrator := mockOrchestrator.NewMockOrchestrator(mockCtrl)
403+
defer func() { osFs = afero.NewOsFs() }()
404+
osFs = afero.NewMemMapFs()
405+
406+
help, _ := NewHelper(orchestrator, "/var/lib/kubelet", false)
407+
h, ok := help.(*helper)
408+
if !ok {
409+
t.Fatal("Could not cast helper to a NodeHelper!")
410+
}
411+
412+
podUUIDPathBase := "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/"
413+
volName1 := "pvc-123"
414+
podUUID1 := "123-456"
415+
podUUIDPath1 := podUUIDPathBase + volName1
416+
_ = osFs.MkdirAll(podUUIDPath1, 0o777)
417+
_, _ = osFs.Create(podUUIDPath1 + "/" + podUUID1)
418+
419+
volName2 := "pvc-123"
420+
podUUID2 := "123-456"
421+
podUUIDPath2 := podUUIDPathBase + volName2
422+
_ = osFs.MkdirAll(podUUIDPath2, 0o777)
423+
_, _ = osFs.Create(podUUIDPath2 + "/" + podUUID2)
424+
425+
mapping := make(map[string]map[string]struct{})
426+
427+
err := h.discoverPVCsToPublishedPathsRawDevices(context.Background(), mapping)
428+
expectedPublishedPath := filepath.Join(podUUIDPath1, podUUID1)
429+
_, ok = mapping[volName1][expectedPublishedPath]
430+
assert.NoError(t, err)
431+
assert.True(t, ok, "expected published path not found in map!")
432+
433+
expectedPublishedPath = filepath.Join(podUUIDPath2, podUUID2)
434+
_, ok = mapping[volName1][expectedPublishedPath]
435+
assert.NoError(t, err)
436+
assert.True(t, ok, "expected published path not found in map!")
437+
}
438+
439+
func TestDiscoverPVCsToPublishedPathsRawDevices_EmptyMap(t *testing.T) {
440+
mockCtrl := gomock.NewController(t)
441+
orchestrator := mockOrchestrator.NewMockOrchestrator(mockCtrl)
442+
defer func() { osFs = afero.NewOsFs() }()
443+
osFs = afero.NewMemMapFs()
444+
445+
help, _ := NewHelper(orchestrator, "/var/lib/kubelet", false)
446+
h, ok := help.(*helper)
447+
if !ok {
448+
t.Fatal("Could not cast helper to a NodeHelper!")
449+
}
450+
451+
podUUIDPathBase := "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/"
452+
volName1 := "pvc-123"
453+
podUUID1 := "123-456"
454+
podUUIDPath1 := podUUIDPathBase + volName1
455+
_ = osFs.MkdirAll(podUUIDPath1, 0o777)
456+
_, _ = osFs.Create(podUUIDPath1 + "/" + podUUID1)
457+
458+
volName2 := "pvc-123"
459+
podUUID2 := "123-456"
460+
podUUIDPath2 := podUUIDPathBase + volName2
461+
_ = osFs.MkdirAll(podUUIDPath2, 0o777)
462+
_, _ = osFs.Create(podUUIDPath2 + "/" + podUUID2)
463+
464+
mapping := make(map[string]map[string]struct{})
465+
466+
// invalid path
467+
h.kubeConfigPath = "/abc/something"
468+
err := h.discoverPVCsToPublishedPathsRawDevices(context.Background(), mapping)
469+
assert.Nil(t, err)
470+
assert.True(t, len(mapping) == 0, "expected empty map!")
471+
}
472+
400473
func newValidHelper(
401474
orchestrator *mockOrchestrator.MockOrchestrator, volId string, mockPubMgr *mockNodeHelpers.MockVolumePublishManager,
402475
) *helper {

‎frontend/csi/node_server.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1173,7 +1173,8 @@ func (p *Plugin) nodeStageISCSIVolume(
11731173
return nil
11741174
}
11751175

1176-
// ensureAttachISCSIVolume to ensure that iSCSI volume attachment is through.
1176+
// ensureAttachISCSIVolume attempts to attach the volume to the local host
1177+
// with a retry logic based on the pubish information passed in.
11771178
func (p *Plugin) ensureAttachISCSIVolume(
11781179
ctx context.Context, req *csi.NodeStageVolumeRequest, mountpoint string,
11791180
publishInfo *utils.VolumePublishInfo, attachTimeout time.Duration,
@@ -1240,7 +1241,7 @@ func (p *Plugin) nodeUnstageISCSIVolume(
12401241
// No need to return an error
12411242
Logc(ctx).WithFields(LogFields{
12421243
"devicePath": publishInfo.DevicePath,
1243-
"LUN": publishInfo.IscsiLunNumber,
1244+
"lun": publishInfo.IscsiLunNumber,
12441245
"err": err,
12451246
}).Error("Failed to verify the multipath device, could not determine" +
12461247
" underlying device for LUKS mapping.")

0 commit comments

Comments
 (0)