diff --git a/integrationtests/disk_test.go b/integrationtests/disk_test.go index aa3578bb..d79ac110 100644 --- a/integrationtests/disk_test.go +++ b/integrationtests/disk_test.go @@ -1,9 +1,197 @@ package integrationtests import ( + "context" + "fmt" + "math/rand" + "strconv" + "strings" "testing" + "time" + + "github.com/kubernetes-csi/csi-proxy/pkg/disk" + diskapi "github.com/kubernetes-csi/csi-proxy/pkg/disk/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// This test is meant to run on GCE where the page83 ID of the first disk contains +// the host name +// Skip on Github Actions as it is expected to fail +func TestDisk(t *testing.T) { + t.Run("ListDiskIDs,ListDiskLocations", func(t *testing.T) { + // even though this test doesn't need the VHD API it failed in Github Actions + // disk_v1beta3_test.go:30: + // Error Trace: disk_v1beta3_test.go:30 + // Error: Expected nil, but got: &status.statusError{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), Code:2, Message:"Could not get page83 ID: IOCTL_STORAGE_QUERY_PROPERTY failed: Incorrect function.", Details:[]*anypb.Any(nil)} + // Test: TestDiskAPIGroup/v1beta3Tests/ListDiskIDs,ListDiskLocations + skipTestOnCondition(t, isRunningOnGhActions()) + + client, err := disk.New(diskapi.New()) + require.Nil(t, err) + + listRequest := &disk.ListDiskIDsRequest{} + diskIDsResponse, err := client.ListDiskIDs(context.TODO(), listRequest) + require.Nil(t, err) + + // example output for GCE (0 is ok, others are virtual disks) + // diskIDs:{key:0 value:{page83:"Google persistent-disk-0" serial_number:" "}} + // diskIDs:{key:1 value:{page83:"4d53465420202020328d59b360875845ac645473be8267bf"}} + // diskIDs:{key:2 value:{page83:"4d534654202020208956a91dadfe3d48865f9b9bcbdb8d3e"}} + // diskIDs:{key:3 value:{page83:"4d534654202020207a3d18d72787ee47bdc127cb4f06403a"}} + t.Logf("diskIDsResponse=%v", diskIDsResponse) + + cmd := "hostname" + hostname, err := runPowershellCmd(t, cmd) + if err != nil { + t.Errorf("Error: %v. Command: %s. Out: %s", err, cmd, hostname) + } + + diskIDsMap := diskIDsResponse.DiskIDs + if len(diskIDsMap) == 0 { + t.Errorf("Expected to get at least one diskIDs, instead got diskIDsResponse.DiskIDs=%+v", diskIDsMap) + } + + // some disks may have the field Page83, if it's a GCE Persistent disk + // it'll have a nonempty SerialNumber + // first disk is the VM disk (other disks might be VHD) + for diskNumber, diskIDs := range diskIDsMap { + if len(diskIDs.SerialNumber) > 0 { + // the nvme disks don't have a Page83 number + if strings.HasPrefix(diskIDs.SerialNumber, "nvme") { + continue + } + page83 := diskIDs.Page83 + if page83 == "" { + t.Errorf("page83 field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs) + } + } + } + + listDiskLocationsRequest := &disk.ListDiskLocationsRequest{} + listDiskLocationsResponse, err := client.ListDiskLocations(context.TODO(), listDiskLocationsRequest) + require.Nil(t, err) + t.Logf("listDiskLocationsResponse=%v", listDiskLocationsResponse) + if len(listDiskLocationsResponse.DiskLocations) == 0 { + t.Errorf("Expected to get at least one diskLocation, instead got DiskLocations=%+v", listDiskLocationsResponse.DiskLocations) + } + }) + + t.Run("Get/SetDiskState", func(t *testing.T) { + skipTestOnCondition(t, isRunningOnGhActions()) + + client, err := disk.New(diskapi.New()) + require.Nil(t, err) + + // initialize disk + vhd, vhdCleanup := diskInit(t) + defer vhdCleanup() + + // disk stats + diskStatsRequest := &disk.GetDiskStatsRequest{ + DiskNumber: vhd.DiskNumber, + } + diskStatsResponse, err := client.GetDiskStats(context.TODO(), diskStatsRequest) + require.NoError(t, err) + if !sizeIsAround(t, diskStatsResponse.TotalBytes, vhd.InitialSize) { + t.Fatalf("DiskStats doesn't have the expected size, wanted (close to)=%d got=%d", vhd.InitialSize, diskStatsResponse.TotalBytes) + } + + // Rescan + _, err = client.Rescan(context.TODO(), &disk.RescanRequest{}) + require.NoError(t, err) + + // change disk state + out, err := runPowershellCmd(t, fmt.Sprintf("Get-Disk -Number %d | Set-Disk -IsOffline $true", vhd.DiskNumber)) + require.NoError(t, err, "failed setting disk offline, out=%v", out) + + getReq := &disk.GetDiskStateRequest{DiskNumber: vhd.DiskNumber} + getResp, err := client.GetDiskState(context.TODO(), getReq) + + if assert.NoError(t, err) { + assert.False(t, getResp.IsOnline, "Expected disk to be offline") + } + + setReq := &disk.SetDiskStateRequest{DiskNumber: vhd.DiskNumber, IsOnline: true} + _, err = client.SetDiskState(context.TODO(), setReq) + assert.NoError(t, err) + + out, err = runPowershellCmd(t, fmt.Sprintf("Get-Disk -Number %d | Select-Object -ExpandProperty IsOffline", vhd.DiskNumber)) + assert.NoError(t, err) + + result, err := strconv.ParseBool(strings.TrimSpace(out)) + assert.NoError(t, err) + assert.False(t, result, "Expected disk to be online") + + getReq = &disk.GetDiskStateRequest{DiskNumber: vhd.DiskNumber} + getResp, err = client.GetDiskState(context.TODO(), getReq) + + if assert.NoError(t, err) { + assert.True(t, getResp.IsOnline, "Expected disk is online") + } + + setReq = &disk.SetDiskStateRequest{DiskNumber: vhd.DiskNumber, IsOnline: false} + _, err = client.SetDiskState(context.TODO(), setReq) + assert.NoError(t, err) + + out, err = runPowershellCmd(t, fmt.Sprintf("Get-Disk -Number %d | Select-Object -ExpandProperty IsOffline", vhd.DiskNumber)) + assert.NoError(t, err) + + result, err = strconv.ParseBool(strings.TrimSpace(out)) + assert.NoError(t, err) + assert.True(t, result, "Expected disk to be offline") + }) + + t.Run("PartitionDisk", func(t *testing.T) { + skipTestOnCondition(t, isRunningOnGhActions()) + + var err error + client, err := disk.New(diskapi.New()) + require.Nil(t, err) + + // initialize disk but don't partition it using `diskInit` + s1 := rand.NewSource(time.Now().UTC().UnixNano()) + r1 := rand.New(s1) + + testPluginPath := fmt.Sprintf("C:\\var\\lib\\kubelet\\plugins\\testplugin-%d.csi.io\\", r1.Intn(100)) + mountPath := fmt.Sprintf("%smount-%d", testPluginPath, r1.Intn(100)) + vhdxPath := fmt.Sprintf("%sdisk-%d.vhdx", testPluginPath, r1.Intn(100)) + + var cmd, out string + const initialSize = 1 * 1024 * 1024 * 1024 + + cmd = fmt.Sprintf("mkdir %s", mountPath) + if out, err = runPowershellCmd(t, cmd); err != nil { + t.Fatalf("Error: %v. Command: %q. Out: %s", err, cmd, out) + } + cmd = fmt.Sprintf("New-VHD -Path %s -SizeBytes %d", vhdxPath, initialSize) + if out, err = runPowershellCmd(t, cmd); err != nil { + t.Fatalf("Error: %v. Command: %q. Out: %s.", err, cmd, out) + } + cmd = fmt.Sprintf("Mount-VHD -Path %s", vhdxPath) + if out, err = runPowershellCmd(t, cmd); err != nil { + t.Fatalf("Error: %v. Command: %q. Out: %s", err, cmd, out) + } + + var diskNum uint64 + var diskNumUnparsed string + cmd = fmt.Sprintf("(Get-VHD -Path %s).DiskNumber", vhdxPath) + if diskNumUnparsed, err = runPowershellCmd(t, cmd); err != nil { + t.Fatalf("Error: %v. Command: %s", err, cmd) + } + if diskNum, err = strconv.ParseUint(strings.TrimRight(diskNumUnparsed, "\r\n"), 10, 32); err != nil { + t.Fatalf("Error: %v", err) + } + + // make disk partition request + diskPartitionRequest := &disk.PartitionDiskRequest{ + DiskNumber: uint32(diskNum), + } + _, err = client.PartitionDisk(context.TODO(), diskPartitionRequest) + require.NoError(t, err) + }) +} + // This test is meant to run on GCE where the page83 ID of the first disk contains // the host name // Skip on Github Actions as it is expected to fail diff --git a/pkg/disk/api/api.go b/pkg/disk/api/api.go new file mode 100644 index 00000000..84f9a9a2 --- /dev/null +++ b/pkg/disk/api/api.go @@ -0,0 +1,367 @@ +package api + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "syscall" + "unsafe" + + "github.com/kubernetes-csi/csi-proxy/pkg/utils" + "k8s.io/klog/v2" +) + +var ( + kernel32DLL = syscall.NewLazyDLL("kernel32.dll") +) + +const ( + IOCTL_STORAGE_GET_DEVICE_NUMBER = 0x2D1080 + IOCTL_STORAGE_QUERY_PROPERTY = 0x002d1400 +) + +// API declares the interface exposed by the internal API +type API interface { + // ListDiskLocations - constructs a map with the disk number as the key and the DiskLocation structure + // as the value. The DiskLocation struct has various fields like the Adapter, Bus, Target and LUNID. + ListDiskLocations() (map[uint32]DiskLocation, error) + // IsDiskInitialized returns true if the disk identified by `diskNumber` is initialized. + IsDiskInitialized(diskNumber uint32) (bool, error) + // InitializeDisk initializes the disk `diskNumber` + InitializeDisk(diskNumber uint32) error + // BasicPartitionsExist checks if the disk `diskNumber` has any basic partitions. + BasicPartitionsExist(diskNumber uint32) (bool, error) + // CreateBasicPartition creates a partition in disk `diskNumber` + CreateBasicPartition(diskNumber uint32) error + // Rescan updates the host storage cache (re-enumerates disk, partition and volume objects) + Rescan() error + // GetDiskNumberByName gets a disk number by page83 ID (disk name) + GetDiskNumberByName(page83ID string) (uint32, error) + // ListDiskIDs list all disks by disk number. + ListDiskIDs() (map[uint32]DiskIDs, error) + // GetDiskStats gets the disk stats of the disk `diskNumber`. + GetDiskStats(diskNumber uint32) (int64, error) + // SetDiskState sets the offline/online state of the disk `diskNumber`. + SetDiskState(diskNumber uint32, isOnline bool) error + // GetDiskState gets the offline/online state of the disk `diskNumber`. + GetDiskState(diskNumber uint32) (bool, error) +} + +// DiskAPI implements the OS API calls related to Disk Devices. All code here should be very simple +// pass-through to the OS APIs or cmdlets. Any logic around the APIs/cmdlet invocation +// should go in pkg/disk/disk.go so that logic can be easily unit-tested +// without requiring specific OS environments. +type DiskAPI struct{} + +// ensure that DiskAPI implements the exposed API +var _ API = &DiskAPI{} + +func New() DiskAPI { + return DiskAPI{} +} + +// ListDiskLocations - constructs a map with the disk number as the key and the DiskLocation structure +// as the value. The DiskLocation struct has various fields like the Adapter, Bus, Target and LUNID. +func (DiskAPI) ListDiskLocations() (map[uint32]DiskLocation, error) { + // sample response + // [{ + // "number": 0, + // "location": "PCI Slot 3 : Adapter 0 : Port 0 : Target 1 : LUN 0" + // }, ...] + cmd := "ConvertTo-Json @(Get-Disk | select Number, Location)" + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return nil, fmt.Errorf("failed to list disk location. cmd: %q, output: %q, err %v", cmd, string(out), err) + } + + var getDisk []map[string]interface{} + err = json.Unmarshal(out, &getDisk) + if err != nil { + return nil, err + } + + m := make(map[uint32]DiskLocation) + for _, v := range getDisk { + str := v["Location"].(string) + num := v["Number"].(float64) + + found := false + s := strings.Split(str, ":") + if len(s) >= 5 { + var d DiskLocation + for _, item := range s { + item = strings.TrimSpace(item) + itemSplit := strings.Split(item, " ") + if len(itemSplit) == 2 { + found = true + switch strings.TrimSpace(itemSplit[0]) { + case "Adapter": + d.Adapter = strings.TrimSpace(itemSplit[1]) + case "Target": + d.Target = strings.TrimSpace(itemSplit[1]) + case "LUN": + d.LUNID = strings.TrimSpace(itemSplit[1]) + default: + klog.Warningf("Got unknown field : %s=%s", itemSplit[0], itemSplit[1]) + } + } + } + + if found { + m[uint32(num)] = d + } + } + } + return m, nil +} + +func (DiskAPI) Rescan() error { + cmd := "Update-HostStorageCache" + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return fmt.Errorf("error updating host storage cache output: %q, err: %v", string(out), err) + } + return nil +} + +func (DiskAPI) IsDiskInitialized(diskNumber uint32) (bool, error) { + cmd := fmt.Sprintf("Get-Disk -Number %d | Where partitionstyle -eq 'raw'", diskNumber) + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return false, fmt.Errorf("error checking initialized status of disk %d: %v, %v", diskNumber, out, err) + } + if len(out) == 0 { + // disks with raw initialization not detected + return true, nil + } + return false, nil +} + +func (DiskAPI) InitializeDisk(diskNumber uint32) error { + cmd := fmt.Sprintf("Initialize-Disk -Number %d -PartitionStyle GPT", diskNumber) + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return fmt.Errorf("error initializing disk %d: %v, %v", diskNumber, out, err) + } + return nil +} + +func (DiskAPI) BasicPartitionsExist(diskNumber uint32) (bool, error) { + cmd := fmt.Sprintf("Get-Partition | Where DiskNumber -eq %d | Where Type -ne Reserved", diskNumber) + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return false, fmt.Errorf("error checking presence of partitions on disk %d: %v, %v", diskNumber, out, err) + } + if len(out) > 0 { + // disk has partitions in it + return true, nil + } + return false, nil +} + +func (DiskAPI) CreateBasicPartition(diskNumber uint32) error { + cmd := fmt.Sprintf("New-Partition -DiskNumber %d -UseMaximumSize", diskNumber) + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return fmt.Errorf("error creating partition on disk %d: %v, %v", diskNumber, out, err) + } + return nil +} + +func (imp DiskAPI) GetDiskNumberByName(page83ID string) (uint32, error) { + diskNumber, err := imp.GetDiskNumberWithID(page83ID) + return diskNumber, err +} + +func (DiskAPI) GetDiskNumber(disk syscall.Handle) (uint32, error) { + var bytes uint32 + devNum := StorageDeviceNumber{} + buflen := uint32(unsafe.Sizeof(devNum.DeviceType)) + uint32(unsafe.Sizeof(devNum.DeviceNumber)) + uint32(unsafe.Sizeof(devNum.PartitionNumber)) + + err := syscall.DeviceIoControl(disk, IOCTL_STORAGE_GET_DEVICE_NUMBER, nil, 0, (*byte)(unsafe.Pointer(&devNum)), buflen, &bytes, nil) + + return devNum.DeviceNumber, err +} + +func (DiskAPI) GetDiskPage83ID(disk syscall.Handle) (string, error) { + query := StoragePropertyQuery{} + + bufferSize := uint32(4 * 1024) + buffer := make([]byte, 4*1024) + var size uint32 + var n uint32 + var m uint16 + + query.QueryType = PropertyStandardQuery + query.PropertyID = StorageDeviceIDProperty + + querySize := uint32(unsafe.Sizeof(query)) + err := syscall.DeviceIoControl(disk, IOCTL_STORAGE_QUERY_PROPERTY, (*byte)(unsafe.Pointer(&query)), querySize, (*byte)(unsafe.Pointer(&buffer[0])), bufferSize, &size, nil) + if err != nil { + return "", fmt.Errorf("IOCTL_STORAGE_QUERY_PROPERTY failed: %v", err) + } + + devIDDesc := (*StorageDeviceIDDescriptor)(unsafe.Pointer(&buffer[0])) + + pID := (*StorageIdentifier)(unsafe.Pointer(&devIDDesc.Identifiers[0])) + + page83ID := []byte{} + byteSize := unsafe.Sizeof(byte(0)) + for n = 0; n < devIDDesc.NumberOfIdentifiers; n++ { + if pID.Association == StorageIDAssocDevice && (pID.CodeSet == StorageIDCodeSetBinary || pID.CodeSet == StorageIDCodeSetASCII) { + for m = 0; m < pID.IdentifierSize; m++ { + page83ID = append(page83ID, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(&pID.Identifier[0])) + byteSize*uintptr(m)))) + } + + if pID.CodeSet == StorageIDCodeSetASCII { + return string(page83ID), nil + } else if pID.CodeSet == StorageIDCodeSetBinary { + return hex.EncodeToString(page83ID), nil + } + } + pID = (*StorageIdentifier)(unsafe.Pointer(uintptr(unsafe.Pointer(pID)) + byteSize*uintptr(pID.NextOffset))) + } + return "", nil +} + +func (imp DiskAPI) GetDiskNumberWithID(page83ID string) (uint32, error) { + cmd := "ConvertTo-Json @(Get-Disk | Select Path)" + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return 0, fmt.Errorf("Could not query disk paths") + } + + outString := string(out) + disks := []Disk{} + err = json.Unmarshal([]byte(outString), &disks) + if err != nil { + return 0, err + } + + for i := range disks { + diskNumber, diskPage83ID, err := imp.GetDiskNumberAndPage83ID(disks[i].Path) + if err != nil { + return 0, err + } + + if diskPage83ID == page83ID { + return diskNumber, nil + } + } + + return 0, fmt.Errorf("Could not find disk with Page83 ID %s", page83ID) +} + +func (imp DiskAPI) GetDiskNumberAndPage83ID(path string) (uint32, string, error) { + h, err := syscall.Open(path, syscall.O_RDONLY, 0) + defer syscall.Close(h) + if err != nil { + return 0, "", err + } + + diskNumber, err := imp.GetDiskNumber(h) + if err != nil { + return 0, "", err + } + + page83ID, err := imp.GetDiskPage83ID(h) + if err != nil { + return 0, "", err + } + + return diskNumber, page83ID, nil +} + +// ListDiskIDs - constructs a map with the disk number as the key and the DiskID structure +// as the value. The DiskID struct has a field for the page83 ID. +func (imp DiskAPI) ListDiskIDs() (map[uint32]DiskIDs, error) { + // sample response + // [ + // { + // "Path": "\\\\?\\scsi#disk\u0026ven_google\u0026prod_persistentdisk#4\u002621cb0360\u00260\u0026000100#{53f56307-b6bf-11d0-94f2-00a0c91efb8b}", + // "SerialNumber": " " + // }, + // { + // "Path": "\\\\?\\scsi#disk\u0026ven_msft\u0026prod_virtual_disk#2\u00261f4adffe\u00260\u0026000001#{53f56307-b6bf-11d0-94f2-00a0c91efb8b}", + // "SerialNumber": null + // }, ] + cmd := "ConvertTo-Json @(Get-Disk | Select Path, SerialNumber)" + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return nil, fmt.Errorf("Could not query disk paths: %v", err) + } + + outString := string(out) + disks := []Disk{} + err = json.Unmarshal([]byte(outString), &disks) + if err != nil { + return nil, err + } + + m := make(map[uint32]DiskIDs) + + for i := range disks { + diskNumber, page83, err := imp.GetDiskNumberAndPage83ID(disks[i].Path) + if err != nil { + return nil, err + } + + m[diskNumber] = DiskIDs{ + Page83: page83, + SerialNumber: disks[i].SerialNumber, + } + } + + return m, nil +} + +func (imp DiskAPI) GetDiskStats(diskNumber uint32) (int64, error) { + cmd := fmt.Sprintf("(Get-Disk -Number %d).Size", diskNumber) + out, err := utils.RunPowershellCmd(cmd) + if err != nil || len(out) == 0 { + return -1, fmt.Errorf("error getting size of disk. cmd: %s, output: %s, error: %v", cmd, string(out), err) + } + + reg, err := regexp.Compile("[^0-9]+") + if err != nil { + return -1, fmt.Errorf("error compiling regex. err: %v", err) + } + diskSizeOutput := reg.ReplaceAllString(string(out), "") + + diskSize, err := strconv.ParseInt(diskSizeOutput, 10, 64) + + if err != nil { + return -1, fmt.Errorf("error parsing size of disk. cmd: %s, output: %s, error: %v", cmd, diskSizeOutput, err) + } + + return diskSize, nil +} + +func (imp DiskAPI) SetDiskState(diskNumber uint32, isOnline bool) error { + cmd := fmt.Sprintf("(Get-Disk -Number %d) | Set-Disk -IsOffline $%t", diskNumber, !isOnline) + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return fmt.Errorf("error setting disk attach state. cmd: %s, output: %s, error: %v", cmd, string(out), err) + } + + return nil +} + +func (imp DiskAPI) GetDiskState(diskNumber uint32) (bool, error) { + cmd := fmt.Sprintf("(Get-Disk -Number %d) | Select-Object -ExpandProperty IsOffline", diskNumber) + out, err := utils.RunPowershellCmd(cmd) + if err != nil { + return false, fmt.Errorf("error getting disk state. cmd: %s, output: %s, error: %v", cmd, string(out), err) + } + + sout := strings.TrimSpace(string(out)) + isOffline, err := strconv.ParseBool(sout) + if err != nil { + return false, fmt.Errorf("error parsing disk state. output: %s, error: %v", sout, err) + } + + return !isOffline, nil +} diff --git a/pkg/disk/api/types.go b/pkg/disk/api/types.go new file mode 100644 index 00000000..f0ba588d --- /dev/null +++ b/pkg/disk/api/types.go @@ -0,0 +1,129 @@ +package api + +type StorageDeviceNumber struct { + DeviceType DeviceType + DeviceNumber uint32 + PartitionNumber uint32 +} +type DeviceType uint32 + +type StoragePropertyID uint32 + +const ( + StorageDeviceProperty StoragePropertyID = iota + StorageAdapterProperty + StorageDeviceIDProperty + StorageDeviceUniqueIDProperty + StorageDeviceWriteCacheProperty + StorageMiniportProperty + StorageAccessAlignmentProperty + StorageDeviceSeekPenaltyProperty + StorageDeviceTrimProperty + StorageDeviceWriteAggregationProperty + StorageDeviceDeviceTelemetryProperty + StorageDeviceLBProvisioningProperty + StorageDevicePowerProperty + StorageDeviceCopyOffloadProperty + StorageDeviceResiliencyProperty + StorageDeviceMediumProductType + StorageAdapterRpmbProperty + StorageAdapterCryptoProperty + StorageDeviceIoCapabilityProperty + StorageAdapterProtocolSpecificProperty + StorageDeviceProtocolSpecificProperty + StorageAdapterTemperatureProperty + StorageDeviceTemperatureProperty + StorageAdapterPhysicalTopologyProperty + StorageDevicePhysicalTopologyProperty + StorageDeviceAttributesProperty + StorageDeviceManagementStatus + StorageAdapterSerialNumberProperty + StorageDeviceLocationProperty + StorageDeviceNumaProperty + StorageDeviceZonedDeviceProperty + StorageDeviceUnsafeShutdownCount + StorageDeviceEnduranceProperty +) + +type StorageQueryType uint32 + +const ( + PropertyStandardQuery StorageQueryType = iota + PropertyExistsQuery + PropertyMaskQuery + PropertyQueryMaxDefined +) + +type StoragePropertyQuery struct { + PropertyID StoragePropertyID + QueryType StorageQueryType + Byte []AdditionalParameters +} + +type AdditionalParameters byte + +type StorageDeviceIDDescriptor struct { + Version uint32 + Size uint32 + NumberOfIdentifiers uint32 + Identifiers [1]byte +} + +type StorageIdentifierCodeSet uint32 + +const ( + StorageIDCodeSetReserved StorageIdentifierCodeSet = iota + StorageIDCodeSetBinary + StorageIDCodeSetASCII + StorageIDCodeSetUtf8 +) + +type StorageIdentifierType uint32 + +const ( + StorageIdTypeVendorSpecific StorageIdentifierType = iota + StorageIDTypeVendorID + StorageIDTypeEUI64 + StorageIDTypeFCPHName + StorageIDTypePortRelative + StorageIDTypeTargetPortGroup + StorageIDTypeLogicalUnitGroup + StorageIDTypeMD5LogicalUnitIdentifier + StorageIDTypeScsiNameString +) + +type StorageAssociationType uint32 + +const ( + StorageIDAssocDevice StorageAssociationType = iota + StorageIDAssocPort + StorageIDAssocTarget +) + +type StorageIdentifier struct { + CodeSet StorageIdentifierCodeSet + Type StorageIdentifierType + IdentifierSize uint16 + NextOffset uint16 + Association StorageAssociationType + Identifier [1]byte +} + +type Disk struct { + Path string `json:"Path"` + SerialNumber string `json:"SerialNumber"` +} + +// DiskLocation definition +type DiskLocation struct { + Adapter string + Bus string + Target string + LUNID string +} + +// DiskIDs definition +type DiskIDs struct { + Page83 string + SerialNumber string +} diff --git a/pkg/disk/disk.go b/pkg/disk/disk.go new file mode 100644 index 00000000..76336513 --- /dev/null +++ b/pkg/disk/disk.go @@ -0,0 +1,238 @@ +package disk + +import ( + "context" + "fmt" + "strconv" + + diskapi "github.com/kubernetes-csi/csi-proxy/pkg/disk/api" + "k8s.io/klog/v2" +) + +type Disk struct { + hostAPI diskapi.API +} + +type Interface interface { + DiskStats(context.Context, *DiskStatsRequest) (*DiskStatsResponse, error) + GetAttachState(context.Context, *GetAttachStateRequest) (*GetAttachStateResponse, error) + GetDiskNumberByName(context.Context, *GetDiskNumberByNameRequest) (*GetDiskNumberByNameResponse, error) + GetDiskState(context.Context, *GetDiskStateRequest) (*GetDiskStateResponse, error) + GetDiskStats(context.Context, *GetDiskStatsRequest) (*GetDiskStatsResponse, error) + ListDiskIDs(context.Context, *ListDiskIDsRequest) (*ListDiskIDsResponse, error) + ListDiskLocations(context.Context, *ListDiskLocationsRequest) (*ListDiskLocationsResponse, error) + PartitionDisk(context.Context, *PartitionDiskRequest) (*PartitionDiskResponse, error) + Rescan(context.Context, *RescanRequest) (*RescanResponse, error) + SetAttachState(context.Context, *SetAttachStateRequest) (*SetAttachStateResponse, error) + SetDiskState(context.Context, *SetDiskStateRequest) (*SetDiskStateResponse, error) +} + +// check that Disk implements Interface +var _ Interface = &Disk{} + +func New(hostAPI diskapi.API) (*Disk, error) { + return &Disk{ + hostAPI: hostAPI, + }, nil +} + +func (d *Disk) ListDiskLocations(context context.Context, request *ListDiskLocationsRequest) (*ListDiskLocationsResponse, error) { + klog.V(2).Infof("Request: ListDiskLocations: %+v", request) + response := &ListDiskLocationsResponse{} + m, err := d.hostAPI.ListDiskLocations() + if err != nil { + klog.Errorf("ListDiskLocations failed: %v", err) + return response, err + } + + response.DiskLocations = make(map[uint32]*DiskLocation) + for k, v := range m { + d := &DiskLocation{} + d.Adapter = v.Adapter + d.Bus = v.Bus + d.Target = v.Target + d.LUNID = v.LUNID + response.DiskLocations[k] = d + } + return response, nil +} + +func (d *Disk) PartitionDisk(context context.Context, request *PartitionDiskRequest) (*PartitionDiskResponse, error) { + klog.V(2).Infof("Request: PartitionDisk with diskNumber=%d", request.DiskNumber) + response := &PartitionDiskResponse{} + diskNumber := request.DiskNumber + + initialized, err := d.hostAPI.IsDiskInitialized(diskNumber) + if err != nil { + klog.Errorf("IsDiskInitialized failed: %v", err) + return response, err + } + if !initialized { + klog.V(4).Infof("Initializing disk %d", diskNumber) + err = d.hostAPI.InitializeDisk(diskNumber) + if err != nil { + klog.Errorf("failed InitializeDisk %v", err) + return response, err + } + } else { + klog.V(4).Infof("Disk %d already initialized", diskNumber) + } + + klog.V(4).Infof("Checking if disk %d has basic partitions", diskNumber) + partitioned, err := d.hostAPI.BasicPartitionsExist(diskNumber) + if err != nil { + klog.Errorf("failed check BasicPartitionsExist %v", err) + return response, err + } + if !partitioned { + klog.V(4).Infof("Creating basic partition on disk %d", diskNumber) + err = d.hostAPI.CreateBasicPartition(diskNumber) + if err != nil { + klog.Errorf("failed CreateBasicPartition %v", err) + return response, err + } + } else { + klog.V(4).Infof("Disk %d already partitioned", diskNumber) + } + return response, nil +} + +func (d *Disk) Rescan(context context.Context, request *RescanRequest) (*RescanResponse, error) { + klog.V(2).Infof("Request: Rescan") + response := &RescanResponse{} + err := d.hostAPI.Rescan() + if err != nil { + klog.Errorf("Rescan failed %v", err) + return nil, err + } + return response, nil +} + +func (d *Disk) GetDiskNumberByName(context context.Context, request *GetDiskNumberByNameRequest) (*GetDiskNumberByNameResponse, error) { + klog.V(4).Infof("Request: GetDiskNumberByName with diskName %q", request.DiskName) + response := &GetDiskNumberByNameResponse{} + diskName := request.DiskName + number, err := d.hostAPI.GetDiskNumberByName(diskName) + if err != nil { + klog.Errorf("GetDiskNumberByName failed: %v", err) + return nil, err + } + response.DiskNumber = number + return response, nil +} + +func (d *Disk) ListDiskIDs(context context.Context, request *ListDiskIDsRequest) (*ListDiskIDsResponse, error) { + klog.V(4).Infof("Request: ListDiskIDs") + + diskIDs, err := d.hostAPI.ListDiskIDs() + if err != nil { + klog.Errorf("ListDiskIDs failed: %v", err) + return nil, err + } + + // Convert from shared to internal type + responseDiskIDs := make(map[uint32]*DiskIDs) + for k, v := range diskIDs { + responseDiskIDs[k] = &DiskIDs{ + Page83: v.Page83, + SerialNumber: v.SerialNumber, + } + } + response := &ListDiskIDsResponse{DiskIDs: responseDiskIDs} + klog.V(5).Infof("Response=%v", response) + return response, nil +} + +func (d *Disk) DiskStats(context context.Context, request *DiskStatsRequest) (*DiskStatsResponse, error) { + klog.V(2).Infof("Request: DiskStats: diskNumber=%d", request.DiskID) + // forward to GetDiskStats + diskNumber, err := strconv.ParseUint(request.DiskID, 10, 64) + if err != nil { + return nil, fmt.Errorf("Failed to format DiskStatsRequest.DiskID with err: %w", err) + } + getDiskStatsRequest := &GetDiskStatsRequest{ + DiskNumber: uint32(diskNumber), + } + getDiskStatsResponse, err := d.GetDiskStats(context, getDiskStatsRequest) + if err != nil { + klog.Errorf("Forward to GetDiskStats failed: %+v", err) + return nil, err + } + return &DiskStatsResponse{ + DiskSize: getDiskStatsResponse.TotalBytes, + }, nil +} + +func (d *Disk) GetDiskStats(context context.Context, request *GetDiskStatsRequest) (*GetDiskStatsResponse, error) { + klog.V(2).Infof("Request: GetDiskStats: diskNumber=%d", request.DiskNumber) + diskNumber := request.DiskNumber + totalBytes, err := d.hostAPI.GetDiskStats(diskNumber) + if err != nil { + klog.Errorf("GetDiskStats failed: %v", err) + return nil, err + } + return &GetDiskStatsResponse{ + TotalBytes: totalBytes, + }, nil +} + +func (d *Disk) SetAttachState(context context.Context, request *SetAttachStateRequest) (*SetAttachStateResponse, error) { + klog.V(2).Infof("Request: SetAttachState: %+v", request) + + // forward to SetDiskState + diskNumber, err := strconv.ParseUint(request.DiskID, 10, 64) + if err != nil { + return nil, fmt.Errorf("Failed to format SetAttachStateRequest.DiskID with err: %w", err) + } + setDiskStateRequest := &SetDiskStateRequest{ + DiskNumber: uint32(diskNumber), + IsOnline: request.IsOnline, + } + _, err = d.SetDiskState(context, setDiskStateRequest) + if err != nil { + klog.Errorf("Forward to SetDiskState failed with: %+v", err) + return nil, err + } + return &SetAttachStateResponse{}, nil +} + +func (d *Disk) SetDiskState(context context.Context, request *SetDiskStateRequest) (*SetDiskStateResponse, error) { + klog.V(2).Infof("Request: SetDiskState with diskNumber=%d and isOnline=%v", request.DiskNumber, request.IsOnline) + err := d.hostAPI.SetDiskState(request.DiskNumber, request.IsOnline) + if err != nil { + klog.Errorf("SetDiskState failed: %v", err) + return nil, err + } + return &SetDiskStateResponse{}, nil +} + +func (d *Disk) GetAttachState(context context.Context, request *GetAttachStateRequest) (*GetAttachStateResponse, error) { + klog.V(2).Infof("Request: GetAttachState: %+v", request) + + // forward to GetDiskState + diskNumber, err := strconv.ParseUint(request.DiskID, 10, 64) + if err != nil { + return nil, fmt.Errorf("Failed to format GetAttachStateRequest.DiskID with err: %w", err) + } + getDiskStateRequest := &GetDiskStateRequest{ + DiskNumber: uint32(diskNumber), + } + getDiskStateResponse, err := d.GetDiskState(context, getDiskStateRequest) + if err != nil { + klog.Errorf("Forward to GetDiskState failed with: %+v", err) + return nil, err + } + return &GetAttachStateResponse{ + IsOnline: getDiskStateResponse.IsOnline, + }, nil +} + +func (d *Disk) GetDiskState(context context.Context, request *GetDiskStateRequest) (*GetDiskStateResponse, error) { + klog.V(4).Infof("Request: GetDiskState with diskNumber=%d", request.DiskNumber) + isOnline, err := d.hostAPI.GetDiskState(request.DiskNumber) + if err != nil { + klog.Errorf("GetDiskState failed with: %v", err) + return nil, err + } + return &GetDiskStateResponse{IsOnline: isOnline}, nil +} diff --git a/pkg/disk/types.go b/pkg/disk/types.go new file mode 100644 index 00000000..05225f88 --- /dev/null +++ b/pkg/disk/types.go @@ -0,0 +1,113 @@ +package disk + +type DiskLocation struct { + Adapter string + Bus string + Target string + LUNID string +} + +type ListDiskLocationsRequest struct { +} + +type ListDiskLocationsResponse struct { + // Map of disk device IDs and associated with each disk device + DiskLocations map[uint32]*DiskLocation +} + +type PartitionDiskRequest struct { + // Disk device ID of the disk to partition + DiskNumber uint32 +} + +type PartitionDiskResponse struct { +} + +type RescanRequest struct { +} + +type RescanResponse struct { +} + +type GetDiskNumberByNameRequest struct { + // Disk name is the page83 ID of the disk + DiskName string +} + +type GetDiskNumberByNameResponse struct { + DiskNumber uint32 +} + +type ListDiskIDsRequest struct { +} + +type DiskIDs struct { + // Map of Disk ID types and Disk ID values + Page83 string + SerialNumber string +} + +type ListDiskIDsResponse struct { + // Map of disk device numbers and IDs associated with each disk device + DiskIDs map[uint32]*DiskIDs +} + +type GetDiskStatsRequest struct { + DiskNumber uint32 +} + +type GetDiskStatsResponse struct { + TotalBytes int64 +} + +type SetDiskStateRequest struct { + // Disk device ID of the disk which state will change + DiskNumber uint32 + + // Online state to set for the disk. true for online, false for offline + IsOnline bool +} + +type SetDiskStateResponse struct { +} + +type GetDiskStateRequest struct { + // Disk device ID of the disk + DiskNumber uint32 +} + +type GetDiskStateResponse struct { + // Online state of the disk. true for online, false for offline + IsOnline bool +} + +// These structs are used in pre v1beta3 API versions + +type DiskStatsRequest struct { + DiskID string +} + +type DiskStatsResponse struct { + DiskSize int64 +} + +type SetAttachStateRequest struct { + // Disk device ID of the disk which state will change + DiskID string + + // Online state to set for the disk. true for online, false for offline + IsOnline bool +} + +type SetAttachStateResponse struct { +} + +type GetAttachStateRequest struct { + // Disk device ID of the disk + DiskID string +} + +type GetAttachStateResponse struct { + // Online state of the disk. true for online, false for offline + IsOnline bool +}