Parallelize iSCSI tests
iSCSI target (=the server) is implemented in Linux kernel. The "iSCSI server" pod is not a real server, it just configures the kernel on the host. In order to run iSCSI tests in parallel, we need to be able to run multiple such pods on a single node, serving different LUNs to different tests. The "server pod" must run with HostNetwork=true to achieve that. Each pod then creates its own IQN with namespace name, so it can't collide with other server pods running in another namespaces on the same node.
This commit is contained in:
Jan Safranek
parent
a93f803f8e
commit
b18dba3794
@ -81,6 +81,9 @@ const (
|
||||
// PodCleanupTimeout is a waiting period for pod to be cleaned up and unmount its volumes so we
|
||||
// don't tear down containers with NFS/Ceph/Gluster server too early.
|
||||
PodCleanupTimeout = 20 * time.Second
|
||||
|
||||
// Template for iSCSI IQN.
|
||||
iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s"
|
||||
)
|
||||
|
||||
// VolumeTestConfig is a struct for configuration of one tests. The test consist of:
|
||||
@ -104,6 +107,8 @@ type VolumeTestConfig struct {
|
||||
ServerVolumes map[string]string
|
||||
// Message to wait for before starting clients
|
||||
ServerReadyMessage string
|
||||
// Use HostNetwork for the server
|
||||
ServerHostNetwork bool
|
||||
// Wait for the pod to terminate successfully
|
||||
// False indicates that the pod is long running
|
||||
WaitForCompletion bool
|
||||
@ -183,20 +188,30 @@ func NewGlusterfsServer(cs clientset.Interface, namespace string) (config Volume
|
||||
}
|
||||
|
||||
// NewISCSIServer is an iSCSI-specific wrapper for CreateStorageServer.
|
||||
func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) {
|
||||
func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip, iqn string) {
|
||||
// Generate cluster-wide unique IQN
|
||||
iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace)
|
||||
|
||||
config = VolumeTestConfig{
|
||||
Namespace: namespace,
|
||||
Prefix: "iscsi",
|
||||
ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer),
|
||||
ServerPorts: []int{3260},
|
||||
ServerArgs: []string{iqn},
|
||||
ServerVolumes: map[string]string{
|
||||
// iSCSI container needs to insert modules from the host
|
||||
"/lib/modules": "/lib/modules",
|
||||
// iSCSI container needs to configure kernel
|
||||
"/sys/kernel": "/sys/kernel",
|
||||
// iSCSI source "block devices" must be available on the host
|
||||
"/srv/iscsi": "/srv/iscsi",
|
||||
},
|
||||
ServerReadyMessage: "Configuration restored from /etc/target/saveconfig.json",
|
||||
ServerReadyMessage: "iscsi target started",
|
||||
ServerHostNetwork: true,
|
||||
}
|
||||
pod, ip = CreateStorageServer(cs, config)
|
||||
return config, pod, ip
|
||||
// Make sure the client runs on the same node as server so we don't need to open any firewalls.
|
||||
config.ClientNodeName = pod.Spec.NodeName
|
||||
return config, pod, ip, iqn
|
||||
}
|
||||
|
||||
// NewRBDServer is a CephRBD-specific wrapper for CreateStorageServer.
|
||||
@ -312,6 +327,7 @@ func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1.
|
||||
},
|
||||
|
||||
Spec: v1.PodSpec{
|
||||
HostNetwork: config.ServerHostNetwork,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: serverPodName,
|
||||
|
||||
@ -330,6 +330,7 @@ type iSCSIVolume struct {
|
||||
serverPod *v1.Pod
|
||||
serverIP string
|
||||
f *framework.Framework
|
||||
iqn string
|
||||
}
|
||||
|
||||
var _ testsuites.TestDriver = &iSCSIDriver{}
|
||||
@ -374,11 +375,10 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, volume tests
|
||||
|
||||
volSource := v1.VolumeSource{
|
||||
ISCSI: &v1.ISCSIVolumeSource{
|
||||
TargetPortal: iv.serverIP + ":3260",
|
||||
// from test/images/volume/iscsi/initiatorname.iscsi
|
||||
IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
|
||||
Lun: 0,
|
||||
ReadOnly: readOnly,
|
||||
TargetPortal: "127.0.0.1:3260",
|
||||
IQN: iv.iqn,
|
||||
Lun: 0,
|
||||
ReadOnly: readOnly,
|
||||
},
|
||||
}
|
||||
if fsType != "" {
|
||||
@ -393,8 +393,8 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, vo
|
||||
|
||||
pvSource := v1.PersistentVolumeSource{
|
||||
ISCSI: &v1.ISCSIPersistentVolumeSource{
|
||||
TargetPortal: iv.serverIP + ":3260",
|
||||
IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
|
||||
TargetPortal: "127.0.0.1:3260",
|
||||
IQN: iv.iqn,
|
||||
Lun: 0,
|
||||
ReadOnly: readOnly,
|
||||
},
|
||||
@ -418,11 +418,13 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes
|
||||
cs := f.ClientSet
|
||||
ns := f.Namespace
|
||||
|
||||
c, serverPod, serverIP := framework.NewISCSIServer(cs, ns.Name)
|
||||
c, serverPod, serverIP, iqn := framework.NewISCSIServer(cs, ns.Name)
|
||||
config.ServerConfig = &c
|
||||
config.ClientNodeName = c.ClientNodeName
|
||||
return &iSCSIVolume{
|
||||
serverPod: serverPod,
|
||||
serverIP: serverIP,
|
||||
iqn: iqn,
|
||||
f: f,
|
||||
}
|
||||
}
|
||||
|
||||
@ -168,7 +168,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
|
||||
init()
|
||||
defer cleanup()
|
||||
|
||||
testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config.ClientNodeSelector)
|
||||
testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config)
|
||||
})
|
||||
}
|
||||
|
||||
@ -176,7 +176,7 @@ func testScriptInPod(
|
||||
f *framework.Framework,
|
||||
volumeType string,
|
||||
source *v1.VolumeSource,
|
||||
nodeSelector map[string]string) {
|
||||
config *PerTestConfig) {
|
||||
|
||||
const (
|
||||
volPath = "/vol1"
|
||||
@ -217,7 +217,8 @@ func testScriptInPod(
|
||||
},
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
NodeSelector: nodeSelector,
|
||||
NodeSelector: config.ClientNodeSelector,
|
||||
NodeName: config.ClientNodeName,
|
||||
},
|
||||
}
|
||||
By(fmt.Sprintf("Creating pod %s", pod.Name))
|
||||
|
||||
@ -16,20 +16,8 @@ FROM BASEIMAGE
|
||||
|
||||
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
|
||||
|
||||
RUN yum install -y iscsi-initiator-utils targetcli net-tools strace procps-ng psmisc && yum clean all
|
||||
ADD run_iscsid.sh /usr/local/bin/
|
||||
ADD initiatorname.iscsi /etc/iscsi/
|
||||
RUN yum install -y targetcli && yum clean all
|
||||
ADD run_iscsi_target.sh /usr/local/bin/
|
||||
ADD block.tar.gz /
|
||||
|
||||
# This JSON file was generated by targetcli with these commands:
|
||||
# /backstores/fileio create block /block
|
||||
# /iscsi create
|
||||
# # Enable demo mode (no authentication!):
|
||||
# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
|
||||
# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1/luns create /backstores/fileio/block
|
||||
# saveconfig
|
||||
ADD saveconfig.json /etc/target/
|
||||
|
||||
EXPOSE 3260/tcp
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/run_iscsid.sh"]
|
||||
ENTRYPOINT ["/usr/local/bin/run_iscsi_target.sh"]
|
||||
|
||||
@ -1 +1 @@
|
||||
1.0
|
||||
2.0
|
||||
|
||||
Binary file not shown.
@ -40,7 +40,7 @@ mkfs.ext2 block
|
||||
|
||||
# Add index.html to it
|
||||
mount -o loop block $MNTDIR
|
||||
echo "Hello from iSCSI" > $MNTDIR/index.html
|
||||
echo "Hello from iscsi" > $MNTDIR/index.html
|
||||
umount $MNTDIR
|
||||
|
||||
rm block.tar.gz 2>/dev/null || :
|
||||
|
||||
@ -1 +0,0 @@
|
||||
InitiatorName=iqn.1994-05.com.redhat:eb59fbe2c4c5
|
||||
72
test/images/volume/iscsi/run_iscsi_target.sh
Executable file
72
test/images/volume/iscsi/run_iscsi_target.sh
Executable file
@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2018 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script does not run any daemon, it only configures iSCSI target (=server)
|
||||
# in kernel. It is possible to run this script multiple times on a single
|
||||
# node, each run will create its own IQN and LUN.
|
||||
|
||||
# Kubernetes must provide unique name.
|
||||
IQN=$1
|
||||
|
||||
# targetcli synchronizes over dbus, however it does not work in
|
||||
# containers. Use flock instead
|
||||
LOCK=/srv/iscsi/targetcli.lock
|
||||
|
||||
function start()
|
||||
{
|
||||
# targetcli need dbus. It may not run on the host, so start a private one
|
||||
mkdir /run/dbus
|
||||
dbus-daemon --system
|
||||
|
||||
# Create new IQN (iSCSI Qualified Name)
|
||||
flock $LOCK targetcli /iscsi create "$IQN"
|
||||
# Run it in demo mode, i.e. no authentication
|
||||
flock $LOCK targetcli /iscsi/"$IQN"/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
|
||||
|
||||
# Create unique "block volume" (i.e. flat file) on the *host*.
|
||||
# Having it in the container confuses kernel from some reason
|
||||
# and it's not able to server multiple LUNs from different
|
||||
# containers.
|
||||
# /srv/iscsi must be bind-mount from the host.
|
||||
cp /block /srv/iscsi/"$IQN"
|
||||
|
||||
# Make the block volume available through our IQN as LUN 0
|
||||
flock $LOCK targetcli /backstores/fileio create block-"$IQN" /srv/iscsi/"$IQN"
|
||||
flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns create /backstores/fileio/block-"$IQN"
|
||||
|
||||
echo "iscsi target started"
|
||||
}
|
||||
|
||||
function stop()
|
||||
{
|
||||
echo "stopping iscsi target"
|
||||
# Remove IQN
|
||||
flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns/ delete 0
|
||||
flock $LOCK targetcli /iscsi delete "$IQN"
|
||||
# Remove block device mapping
|
||||
flock $LOCK targetcli /backstores/fileio delete block-"$IQN"
|
||||
/bin/rm -f /srv/iscsi/"$IQN"
|
||||
echo "iscsi target stopped"
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
||||
trap stop TERM
|
||||
start
|
||||
|
||||
while true; do
|
||||
sleep 1
|
||||
done
|
||||
@ -1,51 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2015 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
function start()
|
||||
{
|
||||
# targetcli need dbus
|
||||
mkdir /run/dbus
|
||||
dbus-daemon --system
|
||||
|
||||
# clear any previous configuration
|
||||
targetcli clearconfig confirm=True
|
||||
|
||||
# restore configuration from saveconfig.json
|
||||
targetcli restoreconfig
|
||||
|
||||
# maximum log level
|
||||
iscsid -f -d 8
|
||||
|
||||
echo "iscsid started"
|
||||
}
|
||||
|
||||
function stop()
|
||||
{
|
||||
echo "Stopping iscsid"
|
||||
killall iscsid
|
||||
targetcli clearconfig confirm=True
|
||||
|
||||
echo "iscsid stopped"
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
||||
trap stop TERM
|
||||
start
|
||||
|
||||
while true; do
|
||||
sleep 5
|
||||
done
|
||||
@ -1,102 +0,0 @@
|
||||
{
|
||||
"fabric_modules": [],
|
||||
"storage_objects": [
|
||||
{
|
||||
"attributes": {
|
||||
"block_size": 512,
|
||||
"emulate_3pc": 1,
|
||||
"emulate_caw": 1,
|
||||
"emulate_dpo": 0,
|
||||
"emulate_fua_read": 0,
|
||||
"emulate_fua_write": 1,
|
||||
"emulate_model_alias": 1,
|
||||
"emulate_rest_reord": 0,
|
||||
"emulate_tas": 1,
|
||||
"emulate_tpu": 0,
|
||||
"emulate_tpws": 0,
|
||||
"emulate_ua_intlck_ctrl": 0,
|
||||
"emulate_write_cache": 1,
|
||||
"enforce_pr_isids": 1,
|
||||
"force_pr_aptpl": 0,
|
||||
"is_nonrot": 0,
|
||||
"max_unmap_block_desc_count": 1,
|
||||
"max_unmap_lba_count": 8192,
|
||||
"max_write_same_len": 4096,
|
||||
"optimal_sectors": 16384,
|
||||
"pi_prot_format": 0,
|
||||
"pi_prot_type": 0,
|
||||
"queue_depth": 128,
|
||||
"unmap_granularity": 1,
|
||||
"unmap_granularity_alignment": 0
|
||||
},
|
||||
"dev": "block",
|
||||
"name": "block",
|
||||
"plugin": "fileio",
|
||||
"size": 126877696,
|
||||
"write_back": true,
|
||||
"wwn": "521c57aa-9d9b-4e5d-ab1a-527487f92a33"
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"fabric": "iscsi",
|
||||
"tpgs": [
|
||||
{
|
||||
"attributes": {
|
||||
"authentication": 0,
|
||||
"cache_dynamic_acls": 1,
|
||||
"default_cmdsn_depth": 64,
|
||||
"default_erl": 0,
|
||||
"demo_mode_discovery": 1,
|
||||
"demo_mode_write_protect": 0,
|
||||
"generate_node_acls": 1,
|
||||
"login_timeout": 15,
|
||||
"netif_timeout": 2,
|
||||
"prod_mode_write_protect": 0,
|
||||
"t10_pi": 0
|
||||
},
|
||||
"enable": true,
|
||||
"luns": [
|
||||
{
|
||||
"index": 0,
|
||||
"storage_object": "/backstores/fileio/block"
|
||||
}
|
||||
],
|
||||
"node_acls": [],
|
||||
"parameters": {
|
||||
"AuthMethod": "CHAP,None",
|
||||
"DataDigest": "CRC32C,None",
|
||||
"DataPDUInOrder": "Yes",
|
||||
"DataSequenceInOrder": "Yes",
|
||||
"DefaultTime2Retain": "20",
|
||||
"DefaultTime2Wait": "2",
|
||||
"ErrorRecoveryLevel": "0",
|
||||
"FirstBurstLength": "65536",
|
||||
"HeaderDigest": "CRC32C,None",
|
||||
"IFMarkInt": "2048~65535",
|
||||
"IFMarker": "No",
|
||||
"ImmediateData": "Yes",
|
||||
"InitialR2T": "Yes",
|
||||
"MaxBurstLength": "262144",
|
||||
"MaxConnections": "1",
|
||||
"MaxOutstandingR2T": "1",
|
||||
"MaxRecvDataSegmentLength": "8192",
|
||||
"MaxXmitDataSegmentLength": "262144",
|
||||
"OFMarkInt": "2048~65535",
|
||||
"OFMarker": "No",
|
||||
"TargetAlias": "LIO Target"
|
||||
},
|
||||
"portals": [
|
||||
{
|
||||
"ip_address": "0.0.0.0",
|
||||
"iser": false,
|
||||
"port": 3260
|
||||
}
|
||||
],
|
||||
"tag": 1
|
||||
}
|
||||
],
|
||||
"wwn": "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -238,7 +238,7 @@ func initImageConfigs() map[int]Config {
|
||||
configs[ServeHostname] = Config{e2eRegistry, "serve-hostname", "1.1"}
|
||||
configs[TestWebserver] = Config{e2eRegistry, "test-webserver", "1.0"}
|
||||
configs[VolumeNFSServer] = Config{e2eRegistry, "volume/nfs", "1.0"}
|
||||
configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "1.0"}
|
||||
configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "2.0"}
|
||||
configs[VolumeGlusterServer] = Config{e2eRegistry, "volume/gluster", "1.0"}
|
||||
configs[VolumeRBDServer] = Config{e2eRegistry, "volume/rbd", "1.0.1"}
|
||||
return configs
|
||||
|
||||
Loading…
Reference in New Issue
Block a user