-
Notifications
You must be signed in to change notification settings - Fork 10
/
ontap-simulator-two-node.sh
executable file
·1191 lines (1069 loc) · 45.8 KB
/
ontap-simulator-two-node.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#install and configure two inode cluster with ontap simulator 9.{7..13}
# |
# |
# .--------------------------|-----,
# | | |
# .----------------, | .----------------, | |
# | .---------, | | | .---------, | | +-----------+
# | | v v v v v | | | | RHEL-N in |
# | | +-------------------+ | | | | bare-metal|
# | | | physical switch | | | | +-----------+
# | | +-------------------+ | | |
# | | | | |
# | | .--------------------|-----,
# | | | | | | |
# | | v | | | |
# | | +-------------------+ | | | +-----------+
# | | | vnet: ontap2-data | | | | | RHEL-N in |
# | | +-------------------+ | | | | KVM |
# | | ^ ^ ^ ^ | | | +-----------+
# | | | | | | | | e.g: vm rhel-8.3% -net=ontap2-data
# +--------------------+ +--------------------+
# |e0f e0c e0d e0e| |e0d e0e e0f e0c|
# | ontap | | ontap |
# | NODE1 | | NODE2 |
# | e0a e0b| |e0a e0b |
# +--------------------+ +--------------------+
# | | | |
# v v v v
# +----------------------+
# | vnet: ontap2-ci |
# +----------------------+
#
command -v vm && test -d /etc/kiss-vm-ns || { echo "[WARN] kiss-vm is required by Ontap Simulator Builder" >&2; exit 127; }
CPID=$$
PROG=$0
ARGS=("$@")
trap_vmpanic() {
echo "[Error] got panic in VM, try again:";
echo '----------------------------------------------------------------'
VMPANIC=yes exec $PROG "${ARGS[@]}";
}
trap trap_vmpanic SIGALRM SIGUSR2
[[ "$VMPANIC" = yes ]] && {
cpuOpt=--cpu=Icelake-Server
PATH=/usr/libexec:$PATH qemu-kvm -cpu ?|grep -q Icelake-Server ||
cpuOpt=--cpu=Skylake-Server
}
# command line parse
P=${0##*/}
Usage() {
cat <<-EOF
Usage:
$P --image <image-file> --license-file <license-file> [otherOPTIONs]
Options:
-h, --help #Display this help.
--image <path> #specify image(ova) file path; e.g: --image=/path/vsim-netapp-DOT9.8-cm_nodar.ova
--license-file <path> #specify the license file path; e.g: --license-file=/path/CMode_licenses_9.8.txt
--dnsaddrs <ip[,ip2]> #e.g: 192.168.10.1 or 192.168.1.1,192.168.2.1
--dnsdomains <domain1[,domain2]> #e.g: test.a.com or test.a.com,devel.a.com
--node1-pubaddr <ip> #node1 management address for public access
--node2-pubaddr <ip> #node2 management address for public access
--lif1-pubaddr <ip> #default lif1.1 address for public access
--lif2-pubaddr <ip> #default lif2.1 address for public access
--no-pubif #don't set public interface, only workaround for nfstest_pnfs test
--vserver-name <NetBIOS> #NetBIOS(or host name) of vserver, used by krb5 configuring
--cifs-workgroup <NetBIOS> #Workgroup Name, This parameter specifies the name of the workgroup (up to 15 characters).
--ad-hostname <FQDN> #Fully Qualified Domain Name, This parameter specifies the name of window servers.
--ad-vm <vmname> #windows servers vm name.
--ad-ip <ip> #windows servers ip.
--ad-admin <user> #Active Directory admin user name
--ad-passwd <passwd> #Active Directory admin password
--ntp-server, --time-server <addr> #ntp/time server hostname/address
--raw #Don't do any pre-configuration, after ONTAP cluster is initialized
EOF
}
#cifs option: https://docs.netapp.com/ontap-9/index.jsp?topic=%2Fcom.netapp.doc.dot-cm-cmpr-910%2Fvserver__cifs__create.html
_at=`getopt -o h \
--long help \
--long image: \
--long license-file: \
--long dnsaddrs: \
--long dnsdomains: \
--long node1-pubaddr: \
--long node2-pubaddr: \
--long lif1-pubaddr: \
--long lif2-pubaddr: \
--long no-pubif \
--long vserver-name: \
--long cifs-workgroup: \
--long ad-hostname: \
--long ad-vm: \
--long ad-ip: \
--long ad-ip-hostonly: \
--long ssh-bind-ip: \
--long ad-admin: \
--long ad-passwd: \
--long ntp-server: --long time-server: \
--long raw \
-a -n "$0" -- "$@"`
[[ $? != 0 ]] && { exit 1; }
eval set -- "$_at"
while true; do
case "$1" in
-h|--help) Usage; shift 1; exit 0;;
--image) ImageFile=$2; shift 2;;
--license-file) LicenseFile=$2; shift 2;;
--dnsaddrs) DNS_ADDRS=$2; shift 2;;
--dnsdomains) DNS_DOMAINS=$2; shift 2;;
--node1-pubaddr) node1_managementif_addr=$2; shift 2;;
--node2-pubaddr) node2_managementif_addr=$2; shift 2;;
--lif1-pubaddr) LIF1_1_ADDR=$2; shift 2;;
--lif2-pubaddr) LIF2_1_ADDR=$2; shift 2;;
--no-pubif) NO_PUBIF=yes; shift 1;;
--vserver-name) NAS_SERVER_NAME=$2; shift 2;;
--cifs-workgroup) CIFS_WORKGROUP=$2; shift 2;;
--ad-hostname) AD_NAME=$2; shift 2;;
--ad-vm) AD_VM=$2; shift 2;;
--ad-ip) AD_IP=$2; shift 2;;
--ad-ip-hostonly) AD_IP_HOSTONLY=$2; SSH_BIND_IP=; shift 2;;
--ssh-bind-ip) SSH_BIND_IP=$2; AD_IP_HOSTONLY=; shift 2;;
--ad-admin) AD_ADMIN=$2; shift 2;;
--ad-passwd) AD_PASSWD=$2; shift 2;;
--ntp-server|--time-server) TIME_SERVER=$2; shift 2;;
--raw) RAW=yes; shift 1;;
--) shift; break;;
esac
done
#__main__
Rundir=/tmp/ontap-simulator-t-$$
mkdir -p $Rundir
clean() { rm -rf $Rundir; }
trap "clean" EXIT
if [[ -z "$ImageFile" || -z "$LicenseFile" ]]; then
Usage >&2
exit 1
fi
if [[ ! -f "$ImageFile" ]]; then
echo "{WARN} image file '${ImageFile}' does not exist." >&2
exit 1
fi
if [[ ! -f $LicenseFile ]]; then
echo "{WARN} license file '${LicenseFile}' does not exist." >&2
exit 1
fi
#get ontap version
_fname=${ImageFile##*/}
ontapver=${_fname#vsim-netapp-DOT}
ontapver=${ontapver%-cm_nodar.ova}
#convert image file to qcow2 files
_dir=$(dirname $ImageFile); [[ ! -w "$_dir" ]] && _dir=/tmp
tar vxf $ImageFile -C $_dir || exit 1
for i in {1..4}; do
qemu-img convert -f vmdk -O qcow2 $_dir/vsim-NetAppDOT-simulate-disk${i}.vmdk $_dir/vsim-NetAppDOT-simulate-disk${i}.qcow2
done
# dependency check
IPCALC=ipcalc; command -v ipcalc-ng &>/dev/null && { IPCALC=ipcalc-ng; }
command -v $IPCALC && command -v nmap || { echo "[WARN] command $IPCALC and nmap is required!" >&2; exit 127; }
freeIpList() {
local nic="$1"
local excludeIpList="$*"
IFS=/ read ip netmasklen < <(get-ip.sh -m $nic)
netaddr=$(get-net-addr.sh $ip/$netmasklen)
local scan_result=$(nmap -v -n -sn $netaddr/$netmasklen 2>/dev/null)
if [[ -n "$excludeIpList" ]]; then
echo "$scan_result" | awk '/host.down/{print $5}' | sed '1d;$d' |
grep -E -v "^${excludeIpList// /|}$"
else
echo "$scan_result" | awk '/host.down/{print $5}' | sed '1d;$d'
fi
}
ExcludeIpList=($AD_IP)
extconnif=$(get-default-nic.sh)
extNetOpt="--net-macvtap=-"
gateWay=$(get-default-gateway.sh)
############################## Assert ##############################
if [[ -n "$AD_IP" ]]; then
echo -e "Assert 1: ping windows AD server: $AD_IP ..."
ping -c 4 $AD_IP || {
if [[ -n "$AD_VM" ]]; then
ipinfo=$(vm exec -v $AD_VM -u "${AD_ADMIN}:${AD_PASSWD}" -- ipconfig)
if ! grep "\<$AD_IP\>" <<<"$ipinfo"; then
exit 1
fi
else
exit 1
fi
}
fi
############################## Assert ##############################
dns_domain_names() {
local _names=$(sed -rn -e '/^search */{s///; s/( |^)local( |$)//; s/ /,/g; p}' /etc/resolv.conf);
test -z "$_names" && _names=$(dnsdomainname)
echo "$_names"
}
dns_addrs() {
local netif="$1" _dnslist=
if grep -q 127.0.0.53 /etc/resolv.conf; then
_dnslist=$(systemd-resolve --status -4 ${netif} |
awk -v RS= 'match($0, /Current DNS Server: ([^\n]+)/, M) {print M[1]}'|paste -sd ,;)
else
_dnslist=$(sed -rn '/^nameserver */{s///; s/ *#.*$//; p}' /etc/resolv.conf | paste -sd ,;)
fi
[[ -n "$_dnslist" ]] && echo $_dnslist || return 1
}
image_binarize() {
local srcf=${1}
local dstf=${2:-new-${srcf}}
if command -v anytopnm >/dev/null; then
anytopnm $srcf | ppmtopgm | pgmtopbm -threshold | pnmtopng > $dstf
else
local ConvertCmd="gm convert"
! command -v gm >/dev/null && {
if ! command -v convert >/dev/null; then
echo "{VM:WARN} command gm or convert are needed by 'vncget' function!" >&2
return 1
else
ConvertCmd=convert
fi
}
$ConvertCmd $srcf -threshold 30% $dstf
fi
return 0
}
if ! which gocr &>/dev/null; then
echo "{WARN} command gocr is needed" >&2
exit 1
fi
vncget() {
local _vncaddr=$1
[[ -z "$_vncaddr" ]] && return 1
vncdo -s ${_vncaddr} capture $Rundir/_screen.png
image_binarize $Rundir/_screen.png $Rundir/_screen2.png || return 1
gocr -i $Rundir/_screen2.png 2>/dev/null
}
colorvncget() { vncget "$@" | GREP_COLORS='ms=01;30;47' grep --color .; }
vncput() {
local vncport=$1
shift
which vncdo >/dev/null || {
echo "{WARN} could not find command 'vncdo'" >&2
return 1
}
[[ -n "$*" ]] && echo -e "\033[1;33m[vncput>$vncport] $*\033[0m"
local msgArray=()
for msg; do
if [[ -n "$msg" ]]; then
if [[ "$msg" = key:* ]]; then
msgArray+=("$msg")
else
regex='[~@#$%^&*()_+|}{":?><!]'
_msg="${msg#type:}"
if [[ "$_msg" =~ $regex ]]; then
while IFS= read -r line; do
[[ "$line" =~ $regex ]] || line="type:$line"
msgArray+=("$line")
done < <(sed -r -e 's;[~!@#$%^&*()_+|}{":?><]+;&\n;g' -e 's;[~!@#$%^&*()_+|}{":?><];\nkey:shift-&;g' <<<"$_msg")
else
msgArray+=("$msg")
fi
fi
msgArray+=("")
else
msgArray+=("$msg")
fi
done
for msg in "${msgArray[@]}"; do
if [[ -n "$msg" ]]; then
if [[ "$msg" = key:* ]]; then
vncdo -s $vncport key "${msg#key:}"
else
vncdo -s $vncport type "${msg#type:}"
fi
else
sleep 1
fi
done
}
vncputln() {
vncput "$@" "key:enter"
}
ocrgrep() {
local pattern=${1}
local ignored_charset="${2:-yaijkfwevy[|:{}}"
pattern=$(sed "s,[${ignored_charset}],.,g" <<<"${pattern,,}")
grep -Ei "${pattern}"
}
vncwait() {
local addr=$1
local pattern="$2"
local tim=${3:-1}
local ignored_charset="$4"
local maxloop=60
local loop=0
local screentext=
echo -e "\n=> waiting: \033[1;36m$pattern\033[0m prompt ..."
screentext=$(vncget $addr)
if echo "$screentext"|grep -E '^(PANIC *:|vpanic)'; then
kill -SIGALRM $CPID
fi
while true; do
vncget $addr | ocrgrep "$pattern" "$ignored_charset" && break
sleep $tim
let loop++
if [[ $loop = $maxloop ]]; then
echo "{WARN}: vncwait has been waiting for more than $(bc <<< "600*$tim") seconds"
screentext=$(vncget $addr)
if echo "$screentext"|grep -E '^(PANIC *:|vpanic)'; then
kill -SIGALRM $CPID
elif echo "$screentext"|grep -E 'Waiting until daemon ktlsd starts up'; then
kill -SIGALRM $CPID
else
echo "$screentext"
fi
loop=0
fi
done
}
vercmp() {
[ $# != 3 ] && {
usage
return 1
}
vl=$1
cmpType=$2
vr=$3
res=1
[ "$vl" = "$vr" ] && eq=1
vmax=$(echo -e "$vl\n$vr" | sort -V | tail -n 1)
case "$cmpType" in
=|eq) [ "$eq" = 1 ] && res=0;;
\>|gt) [ "$eq" != 1 -a "$vl" = "$vmax" ] && res=0;;
\<|lt) [ "$eq" != 1 -a "$vr" = "$vmax" ] && res=0;;
\>=|ge) [ "$vl" = "$vmax" ] && res=0;;
\<=|le) [ "$vr" = "$vmax" ] && res=0;;
*) echo "$vl" | grep -E -q "$vr"; res=$?;;
esac
return $res
}
:; echo -e "\n\033[1;30m================================================================================\033[0m"
:; echo -e "\033[1;30m=> creating networks ...\033[0m"
netcluster=ontap2-ci #e0a e0b
vm netcreate netname=$netcluster brname=br-ontap2-ci forward=
vm netls | grep -w $netcluster >/dev/null || vm netstart $netcluster
netdata=ontap2-data #e0d #e0e
vm netcreate netname=$netdata brname=br-ontap2-data subnet=20
vm netls | grep -w $netdata >/dev/null || vm netstart $netdata
[[ -d /sys/class/net/$extconnif/wireless ]] && {
extconnif=virbr-kissalt
extNetOpt="--net=kissaltnet"
gateWay=$(get-ip.sh $extconnif)
}
[[ -n "$NO_PUBIF" ]] && {
extconnif=br-ontap2-data
extNetOpt=--net=$netdata
gateWay=$(get-ip.sh $extconnif)
}
#===============================================================================
#cluster
cluster_name=fsqe-2nc1
password=fsqe2020
TIME_SERVER=${TIME_SERVER:-time.windows.com}
#===============================================================================
#node1
vmnode1=ontap-node1
node1_managementif_port=e0c
node1_managementif_addr=$node1_managementif_addr
node1_managementif_mask=$(get-net-mask.sh $(get-ip.sh -m $extconnif))
node1_managementif_gateway=$gateWay
cluster_managementif_port=e0d
cluster_managementif_addr=192.168.20.11
cluster_managementif_mask=255.255.255.0
cluster_managementif_gateway=192.168.20.1
dns_domains=$(dns_domain_names)
[[ -n "$DNS_DOMAINS" && $dns_domains != ${DNS_DOMAINS},* ]] && dns_domains=${DNS_DOMAINS},${dns_domains}
dns_domains=$(echo "${dns_domains}"|awk -F, -v OFS=, '{if(NF>3) {print $1,$2,$3} else print}')
dns_addrs=$(dns_addrs $extconnif||dns_addrs)
[[ -n "$DNS_ADDRS" && $dns_addrs != ${DNS_ADDRS},* ]] && dns_addrs=${DNS_ADDRS},${dns_addrs}
dns_addrs=$(echo "${dns_addrs}"|awk -F, -v OFS=, '{if(NF>3) {print $1,$3,$3} else print}')
dns_addrs=${dns_addrs%,}
read controller_located _ < <(hostname -A)
test -z "$controller_located" && read controller_located _ < <(hostname)
:; echo -e "\n\033[1;30m================================================================================\033[0m"
:; echo -e "\033[1;30m=> [$vmnode1] start ...\033[0m"
OSV=freebsd11.2
vm create -n $vmnode1 ONTAP-simulator -i $_dir/vsim-NetAppDOT-simulate-disk1.qcow2 \
--diskbus=ide \
--disk=$_dir/vsim-NetAppDOT-simulate-disk{2..4}.qcow2,bus=ide \
--net=$netcluster,e1000 --net=$netcluster,e1000 \
${extNetOpt},e1000 \
--net=$netdata,e1000 --net=$netdata,e1000 \
${extNetOpt},e1000 \
--noauto --nocloud --video auto --osv $OSV \
--msize $((6*1024)) --cpus 2,cores=2 \
--vncput-after-install key:enter --force $cpuOpt
read vncaddr <<<"$(vm vnc $vmnode1)"
vncaddr=${vncaddr/:/::}
[[ -z "$vncaddr" ]] && {
echo "{WARN}: something is wrong, exit ..." >&2
exit 1
}
echo; expect -c "spawn virsh console $vmnode1
set timeout 8
expect {
-exact {Hit [Enter] to boot immediately} { send \"\\r\"; send_user \" #exit#\\n\"; exit }
{cryptomod_fips:} { send_user \" #exit#\\n\"; exit }
}"
vncwait ${vncaddr} "^login:" 5
[[ -z "$node1_managementif_addr" ]] &&
node1_managementif_addr=$(vncget $vncaddr | sed -nr '/^.*https:..([0-9.]+).*$/{s//\1/; p}')
[[ -z "$node1_managementif_addr" ]] &&
node1_managementif_addr=$(freeIpList $extconnif "${ExcludeIpList[@]}"|sort -R|tail -1)
if [[ -z "$node1_managementif_addr" ]]; then
node1_managementif_addr=169.254.20.11
node1_managementif_mask=16
node1_managementif_gateway=169.254.20.1
fi
ExcludeIpList+=($node1_managementif_addr)
vncputln ${vncaddr} "admin" ""
vncputln ${vncaddr} "reboot"
vncwait ${vncaddr} ".re you sure you want to reboot node.*? .y.n.:" 5
vncputln ${vncaddr} "y"
: <<'COMM'
vncwait ${vncaddr} "Press Ctrl-C for Boot Menu." 5
vncput ${vncaddr} key:ctrl-c
vncwait ${vncaddr} "Selection (1-9)?" 5
vncputln ${vncaddr} "4"
vncwait ${vncaddr} "Zero disks, reset config and install a new file system?" 5
vncputln ${vncaddr} "yes"
vncwait ${vncaddr} "This will erase all the data on the disks, are you sure?" 5
vncputln ${vncaddr} "yes"
COMM
echo "{debug} ontapver: $ontapver"
if [[ "$ontapver" != 9.13.1 ]]; then
echo; expect -c "spawn virsh console $vmnode1
set timeout 120
expect {
-exact {Hit [Enter] to boot immediately} { send \"\\r\"; send_user \" #exit#\\n\"; exit }
{cryptomod_fips:} { send_user \" #exit#\\n\"; exit }
}"
vncwait ${vncaddr} "Type yes to confirm and continue {yes}:" 10
vncputln ${vncaddr} "yes"
vncwait ${vncaddr} "Enter the node management interface port" 2
vncputln ${vncaddr} "${node1_managementif_port}"
vncwait ${vncaddr} "Enter the node management interface .. address" 2
vncputln ${vncaddr} "$node1_managementif_addr"
vncwait ${vncaddr} "Enter the node management interface netmask" 2
vncputln ${vncaddr} "$node1_managementif_mask"
vncwait ${vncaddr} "Enter the node management interface default gateway" 2
vncputln ${vncaddr} "$node1_managementif_gateway"
vncwait ${vncaddr} "complete cluster setup using the command line" 2
vncputln ${vncaddr}
vncwait ${vncaddr} "create a new cluster or join an existing cluster?" 2
vncputln ${vncaddr} "create"
vncwait ${vncaddr} "used as a single node cluster?" 2
vncputln ${vncaddr} "no"
vncwait ${vncaddr} "Do you want to use this configuration?" 2
node1_private_ips=$(vncget $vncaddr|sed -nr '/^.*(169.254.[0-9]+.[0-9]+).*$/{s//\1/; p}'|grep -v '169\.254\.20\.')
vncputln ${vncaddr} "yes"
vncwait ${vncaddr} "administrator.* password:" 2
vncputln ${vncaddr} "$password"
vncwait ${vncaddr} "Retype the password:" 2
vncputln ${vncaddr} "$password"
vncwait ${vncaddr} "Enter the cluster name:" 2
vncputln ${vncaddr} "$cluster_name"
vncwait ${vncaddr} "Enter an additional license key" 2
vncputln ${vncaddr}
vncwait ${vncaddr} "Enter the cluster management interface port" 2
vncputln ${vncaddr} "${cluster_managementif_port}"
vncwait ${vncaddr} "Enter the cluster management interface .. address" 2
vncputln ${vncaddr} "$cluster_managementif_addr"
vncwait ${vncaddr} "Enter the cluster management interface netmask" 2
vncputln ${vncaddr} "$cluster_managementif_mask"
vncwait ${vncaddr} "Enter the cluster management interface default gateway" 2
vncputln ${vncaddr} "$cluster_managementif_gateway"
vncwait ${vncaddr} "Enter the DNS domain names" 2
vncputln ${vncaddr} "$dns_domains"
vncwait ${vncaddr} "Enter the name server .. addresses" 2
vncputln ${vncaddr} "$dns_addrs"
#workaround for DNS verify issue on 9.15.1
while ! vncget ${vncaddr} |
ocrgrep "Where is the controller located|Error: Failed to verify the specified DNS configuration" 'ijkfwevy:{}['; do
sleep 5;
done
vncget ${vncaddr} | ocrgrep "Error: Failed to verify the specified DNS configuration" && {
vncwait ${vncaddr} "Enter the DNS domain names" 2
vncputln ${vncaddr} ""
vncwait ${vncaddr} "Enter the name server .. addresses" 2
vncputln ${vncaddr} ""
}
vncwait ${vncaddr} "Where is the controller located" 2
vncputln ${vncaddr} "$controller_located"
sleep 2
else
echo
expect -c 'spawn virsh console '"$vmnode1"'
set timeout 120
expect {
-exact {Hit [Enter] to boot immediately} { send "\r"; }
{cryptomod_fips:} { send_user " #missing Hit ...#\n"; }
}
set timeout 1200
expect -exact {Type yes to confirm and continue {yes}:} { send "yes\r"; }
expect -re "Enter the node management interface port" { send "'${node1_managementif_port}'\r"; }
expect -re "Enter the node management interface .. address" { send "'$node1_managementif_addr'\r"; }
expect -re "Enter the node management interface netmask" { send "'$node1_managementif_mask'\r"; }
expect -re "Enter the node management interface default gateway" { send "'$node1_managementif_gateway'\r"; }
expect -re "complete cluster setup using the command line" { send "\r"; }
expect "create a new cluster or join an existing cluster?" { send "create\r"; }
expect "used as a single node cluster?" { send "no\r"; }
expect "Do you want to use this configuration?" { send "yes\r"; }
expect -re "administrator.* password:" { send "'$password'\r"; }
expect "Retype the password:" { send "'$password'\r"; }
expect "Enter the cluster name:" { send "'$cluster_name'\r"; }
expect "Enter an additional license key" { send "\r"; }
expect "Enter the cluster management interface port" { send "'${cluster_managementif_port}'\r"; }
expect -re "Enter the cluster management interface .. address" { send "'$cluster_managementif_addr'\r"; }
expect "Enter the cluster management interface netmask" { send "'$cluster_managementif_mask'\r"; }
expect "Enter the cluster management interface default gateway" { send "'$cluster_managementif_gateway'\r"; }
expect "Enter the DNS domain names" { send "'$dns_domains'\r"; }
expect -re "Enter the name server .. addresses" { send "'$dns_addrs'\r"; }
expect "Where is the controller located" { send "'$controller_located'\r"; }
sleep 2
expect "*\r" { send_user "#exit#\r"; }
exit
' > >(tee /tmp/.ontap2-std-console.log)
node1_private_ips=$(sed -nr '/^.*(169.254.[0-9]+.[0-9]+).*$/{s//\1/; p}' /tmp/.ontap2-std-console.log|grep -v '169\.254\.20\.')
fi
:; echo -e "\n\033[1;36m--------------------------------------------------------------------------------\033[0m"
colorvncget $vncaddr
:; echo -e "\n\033[1;36m--------------------------------------------------------------------------------\033[0m"
:; echo -e "\n\033[1;36m=> now ssh(admin@$node1_managementif_addr and admin@$cluster_managementif_addr) is available,\n please complete other configurations in ssh session ...\033[0m"
#===============================================================================
#node2
vmnode2=ontap-node2
node2_managementif_port=e0c
node2_managementif_addr=$node2_managementif_addr
node2_managementif_mask=$(get-net-mask.sh $(get-ip.sh -m $extconnif))
node2_managementif_gateway=$gateWay
:; echo -e "\n\033[1;30m================================================================================\033[0m"
:; echo -e "\033[1;30m=> [$vmnode2] start ...\033[0m"
vm create -n $vmnode2 ONTAP-simulator -i $_dir/vsim-NetAppDOT-simulate-disk1.qcow2 \
--diskbus=ide \
--disk=$_dir/vsim-NetAppDOT-simulate-disk{2..4}.qcow2,bus=ide \
--net=$netcluster,e1000 --net=$netcluster,e1000 \
$extNetOpt,e1000 \
--net=$netdata,e1000 --net=$netdata,e1000 \
$extNetOpt,e1000 \
--noauto --nocloud --video auto --osv $OSV \
--msize $((6*1024)) --cpus 2,cores=2 \
--vncput-after-install "x" --force $cpuOpt
read vncaddr <<<"$(vm vnc $vmnode2)"
vncaddr=${vncaddr/:/::}
[[ -z "$vncaddr" ]] && {
echo "{WARN}: something is wrong, exit ..." >&2
exit 1
}
vncwait ${vncaddr} "VLO.DER>" 0.5
vncputln ${vncaddr} "setenv SYS_SERIAL_NUM 4034389-06-2"
vncputln ${vncaddr} "setenv bootarg.nvram.sysid 4034389062"
vncputln ${vncaddr} "printenv SYS_SERIAL_NUM"
vncputln ${vncaddr} "printenv bootarg.nvram.sysid"
vncputln ${vncaddr} "boot"
vncwait ${vncaddr} "^login:" 5
[[ -z "$node2_managementif_addr" ]] &&
node2_managementif_addr=$(vncget $vncaddr | sed -nr '/^.*https:..([0-9.]+).*$/{s//\1/; p}')
[[ -z "$node2_managementif_addr" ]] &&
node2_managementif_addr=$(freeIpList $extconnif "${ExcludeIpList[@]}"|sort -R|tail -1)
if [[ -z "$node2_managementif_addr" ]]; then
node2_managementif_addr=169.254.20.12
node2_managementif_mask=16
node2_managementif_gateway=169.254.20.1
fi
ExcludeIpList+=($node2_managementif_addr)
vncputln ${vncaddr} "admin" ""
vncputln ${vncaddr} "reboot"
vncwait ${vncaddr} ".re you sure you want to reboot node.*? .y.n.:" 5
vncputln ${vncaddr} "y"
: <<'COMM'
vncwait ${vncaddr} "Press Ctrl-C for Boot Menu." 5
vncput ${vncaddr} key:ctrl-c
vncwait ${vncaddr} "Selection (1-9)?" 5
vncputln ${vncaddr} "4"
vncwait ${vncaddr} "Zero disks, reset config and install a new file system?" 5
vncputln ${vncaddr} "yes"
vncwait ${vncaddr} "This will erase all the data on the disks, are you sure?" 5
vncputln ${vncaddr} "yes"
COMM
if [[ "$ontapver" != 9.13.1 ]]; then
echo; expect -c "spawn virsh console $vmnode2
set timeout 120
expect {
-exact {Hit [Enter] to boot immediately} { send \"\\r\"; send_user \" #exit#\\n\"; exit }
{cryptomod_fips:} { send_user \" #exit#\\n\"; exit }
}"
vncwait ${vncaddr} "Type yes to confirm and continue {yes}:" 10
vncputln ${vncaddr} "yes"
vncwait ${vncaddr} "Enter the node management interface port" 2
vncputln ${vncaddr} "${node2_managementif_port}"
vncwait ${vncaddr} "Enter the node management interface .. address" 2
vncputln ${vncaddr} "$node2_managementif_addr"
vncwait ${vncaddr} "Enter the node management interface netmask" 2
vncputln ${vncaddr} "$node2_managementif_mask"
vncwait ${vncaddr} "Enter the node management interface default gateway" 2
vncputln ${vncaddr} "$node2_managementif_gateway"
vncwait ${vncaddr} "complete cluster setup using the command line" 2
vncputln ${vncaddr}
vncwait ${vncaddr} "create a new cluster or join an existing cluster?" 2
vncputln ${vncaddr} "join"
vncwait ${vncaddr} "Do you want to use this configuration?" 2
vncputln ${vncaddr} "yes"
vncwait ${vncaddr} "cluster you want to join:" 2
read node1_private_ip <<<"$node1_private_ips"
vncputln ${vncaddr} "$node1_private_ip"
#vncwait ${vncaddr} "Enter the username of a user with the .admin. role" 2
vncputln ${vncaddr} "admin"
#vncwait ${vncaddr} "Enter the user's password:" 2
vncputln ${vncaddr} "$password"
vncwait ${vncaddr} "This node has been joined to cluster" 2
else
read node1_private_ip <<<"$node1_private_ips"
expect -c 'spawn virsh console '"$vmnode2"'
expect {
-exact {Hit [Enter] to boot immediately} { send "\r"; }
{cryptomod_fips:} { send_user " #missing Hit ...#\n"; }
}
set timeout 1200
expect -exact {Type yes to confirm and continue {yes}:} { send "yes\r"; }
expect -re "Enter the node management interface port" { send "'${node2_managementif_port}'\r"; }
expect -re "Enter the node management interface .. address" { send "'$node2_managementif_addr'\r"; }
expect -re "Enter the node management interface netmask" { send "'$node2_managementif_mask'\r"; }
expect -re "Enter the node management interface default gateway" { send "'$node2_managementif_gateway'\r"; }
expect -re "complete cluster setup using the command line" { send "\r"; }
expect "create a new cluster or join an existing cluster?" { send "join\r"; }
expect "Do you want to use this configuration?" { send "yes\r"; }
expect "cluster you want to join:" { send "'$node1_private_ip'\r"; }
expect "This node has been joined to cluster" { send_user "#exit#\r"; }
exit
'
fi
:; echo -e "\n\033[1;36m--------------------------------------------------------------------------------\033[0m"
colorvncget $vncaddr
:; echo -e "\n\033[1;36m--------------------------------------------------------------------------------\033[0m"
:; echo -e "\n\033[1;36m=> now ssh(admin@$node1_managementif_addr, admin@$node2_managementif_addr and admin@$cluster_managementif_addr) is available,\n please complete other configurations in ssh session ...\033[0m"
idx=1
for vmnode in $vmnode1 $vmnode2; do
read vncaddr <<<"$(vm vnc $vmnode)"
vncaddr=${vncaddr/:/::}
:; echo -e "\n\033[1;30m================================================================================\033[0m"
:; echo -e "\033[1;30m=> [$vmnode] Delete snapshots and add disk shelf ...\033[0m"
port-available.sh $cluster_managementif_addr 22 --wait
nodename=${cluster_name}-0$idx
diagpasswd=d1234567
expect -c "spawn ssh admin@$cluster_managementif_addr
set timeout 120
expect {Password:} { send \"${password}\\r\" }
expect {${cluster_name}::>} { send \"run -node ${nodename}\\r\" }
expect {${nodename}>} { send \"snap delete -a -f vol0\\r\" }
expect {${nodename}>} { send \"snap sched vol0 0 0 0\\r\" }
expect {${nodename}>} { send \"snap autodelete vol0 on\\r\" }
expect {${nodename}>} { send \"snap autodelete vol0 target_free_space\\r\" }
expect {${nodename}>} { send \"snap autodelete vol0\\r\" }
expect {${nodename}>} { send \"exit\\r\" }
expect {${cluster_name}::>} { send \"security login unlock -username diag\\r\" }
expect {${cluster_name}::>} { send \"security login password -username diag\\r\" }
expect {Enter a new password:} { send \"$diagpasswd\\r\" }
expect {Enter it again:} { send \"$diagpasswd\\r\" }
expect {${cluster_name}::>} { send \"\\r\" }
expect {${cluster_name}::>} { send \"set -privilege diag\\r\" }
expect {Do you want to continue? {y|n}:} { send \"y\\r\" }
expect {${cluster_name}::*>} { send \"systemshell -node ${nodename}\\r\" }
expect -re {diag@[0-9.]+'s password:} { send \"${diagpasswd}\\r\" }
expect {${nodename}%} { send {setenv PATH \"\${PATH}:/usr/sbin\"}; send \"\\r\" }
expect {${nodename}%} { send \"echo \\\$PATH\\r\" }
expect {${nodename}%} { send \"cd /sim/dev\\r\" }
expect {${nodename}%} { send \"ls ,disks/\\r\" }
expect {${nodename}%} { send \"vsim_makedisks -h\\r\" }
expect {${nodename}%} { send \"sudo vsim_makedisks -n 14 -t 37 -a 2\\r\" }
expect {${nodename}%} { send \"sudo vsim_makedisks -n 14 -t 37 -a 3\\r\" }
expect {${nodename}%} { send \"exit\\r\" }
expect {${cluster_name}::*>} { send \"system node reboot -node ${nodename} -ignore-quorum-warnings\\r\" }
expect {Are you sure you want to reboot node} { send \"y\\r\"}
set timeout 10
expect eof
"
if [[ "$ontapver" != 9.13.1 ]]; then
echo; expect -c "spawn virsh console $vmnode
set timeout 120
expect {
-exact {Hit [Enter] to boot immediately} { send \"\\r\"; send_user \" #exit#\\n\"; exit }
{cryptomod_fips:} { send_user \" #exit#\\n\"; exit }
}"
vncwait ${vncaddr} "^login:" 1
vncputln ${vncaddr} "admin"
vncputln ${vncaddr} "${password}"
vncwait ${vncaddr} "${cluster_name}::>" 1
while true; do
vncputln ${vncaddr} "cluster show"
vncget ${vncaddr} | grep "$nodename *true" && break
sleep 5
done
vncputln ${vncaddr} "exit"
else
expect -c 'spawn virsh console '$vmnode'
set timeout 120
expect {
-exact {Hit [Enter] to boot immediately} { send "\r"; }
{cryptomod_fips:} { send_user " #missing Hit ...#\n"; }
}
set timeout 300
expect {login:} { send "admin\r"; }
expect {*:} { send "'${password}'\r"; }
expect {'${cluster_name}'::>} { send "cluster show\r"; }
expect {'$nodename' *true} { sleep 1; }
exit
'
fi
let idx++
done
for ((I=1; I <= 2; I++)); do
nodename=${cluster_name}-0$I
aggr0name=aggr0_${nodename//-/_}
expect -c "spawn ssh admin@$cluster_managementif_addr
set timeout 120
expect {Password:} { send \"${password}\\r\" }
expect {${cluster_name}::>} { send \"disk assign -all true -node ${nodename}\\r\" }
expect {${cluster_name}::>} {
send \"aggr add-disks -aggregate $aggr0name -diskcount 5\\r\"
send \"y\\r\"
send \"y\\r\"
}
while 1 {
expect {${cluster_name}::>} { send \"aggr show -aggregate $aggr0name -fields size\\r\" }
expect {
{*GB} break
{*MB} { sleep 2; continue }
}
}
expect {${cluster_name}::>} { send \"vol modify -vserver ${nodename} -volume vol0 -size 4G\\r\" }
expect {${cluster_name}::>} { send \"exit\\r\" }
expect eof
"
done
#don't do any pre-configuration after system initialization
if [[ -n "$RAW" ]]; then
expect -c "spawn ssh admin@$cluster_managementif_addr
set timeout 120
expect {Password:} { send \"${password}\\r\" }
expect {${cluster_name}::>} { send \"aggr show\\r\" }
expect {${cluster_name}::>} { send \"vol show\\r\" }
expect {${cluster_name}::>} { send \"network port show\\r\" }
expect {${cluster_name}::>} { send \"network interface show\\r\" }
expect {${cluster_name}::>} { send \"exit\\r\" }
expect eof
"
exit
fi
for ((I=1; I <= 2; I++)); do
nodename=${cluster_name}-0$I
expect -c "spawn ssh admin@$cluster_managementif_addr
set timeout 120
expect {Password:} { send \"${password}\\r\" }
expect {${cluster_name}::>} {
send \"aggr create -aggregate aggr${I}_1 -node ${nodename} -disksize 9 -diskcount 28\\r\"
send \"y\\r\"
expect {Job succeeded: DONE} {}
}
expect {${cluster_name}::>} {
send \"aggr create -aggregate aggr${I}_2 -node ${nodename} -disksize 1 -diskcount 20\\r\"
send \"y\\r\"
expect {Job succeeded: DONE} {}
}
expect {${cluster_name}::>} { send \"aggr show\\r\" }
expect {${cluster_name}::>} { send \"exit\\r\" }
expect eof
"
done
getBaseLicense() { local lf=$1; awk 'BEGIN{RS="[\x0d\x0a\x0d]"} /Cluster Base license/ {printf $NF}' $lf; }
getFirstNodeLicenses() { local lf=$1; awk '$2 ~ /^[A-Z]{28}$/ && $2 ~ /ABG/ {print $2}' $lf | paste -sd,; }
getSecondNodeLicenses() { local lf=$1; awk '$2 ~ /^[A-Z]{28}$/ && $2 ~ /EZF/ {print $2}' $lf | paste -sd,; }
BaseLicense=$(getBaseLicense $LicenseFile)
FirstNodeLicenses=$(getFirstNodeLicenses $LicenseFile)
SecondNodeLicenses=$(getSecondNodeLicenses $LicenseFile)
LicenseList=$BaseLicense,$FirstNodeLicenses,$SecondNodeLicenses
expect -c "spawn ssh admin@$cluster_managementif_addr
set timeout 120
expect {Password:} { send \"${password}\\r\" }
expect {${cluster_name}::>} { send \"system license add -license-code $LicenseList\\r\" }
expect {${cluster_name}::>} { send \"aggr show\\r\" }
expect {${cluster_name}::>} { send \"vol show\\r\" }
expect {${cluster_name}::>} { send \"network port show\\r\" }
expect {${cluster_name}::>} { send \"network interface show\\r\" }
expect {${cluster_name}::>} { send \"exit\\r\" }
expect eof
"
VS=vs1
VS_AGGR=aggr1_1
PolicyName=fs_export
testIp=$(get-ip.sh $extconnif)
VOL1=vol1
VOL1_AGGR=aggr1_1
VOL1_SIZE=90G
VOL1_JUNCTION_PATH=/share1
LIF1_0_NAME=lif1.0
LIF1_0_ADDR=192.168.20.21
LIF1_0_MASK=255.255.255.0
LIF1_0_NODE=${cluster_name}-01
LIF1_0_PORT=e0e
LIF1_1_NAME=lif1.1
[[ -z "$LIF1_1_ADDR" ]] && LIF1_1_ADDR=$(freeIpList $extconnif "${ExcludeIpList[@]}"|sort -R|head -1)
ExcludeIpList+=($LIF1_1_ADDR)
LIF1_1_MASK=$(get-net-mask.sh $(get-ip.sh -m $extconnif))
LIF1_1_NODE=${cluster_name}-01
LIF1_1_PORT=e0f
VOL2=vol2
VOL2_AGGR=aggr2_1
VOL2_SIZE=90G
VOL2_JUNCTION_PATH=/share2
LIF2_0_NAME=lif2.0
LIF2_0_ADDR=192.168.20.22
LIF2_0_MASK=255.255.255.0
LIF2_0_NODE=${cluster_name}-02
LIF2_0_PORT=e0e
LIF2_1_NAME=lif2.1
[[ -z "$LIF2_1_ADDR" ]] && LIF2_1_ADDR=$(freeIpList $extconnif "${ExcludeIpList[@]}"|sort -R|head -1)
ExcludeIpList+=($LIF2_1_ADDR)
LIF2_1_MASK=$(get-net-mask.sh $(get-ip.sh -m $extconnif))
LIF2_1_NODE=${cluster_name}-02
LIF2_1_PORT=e0f
[[ -z "$NAS_SERVER_NAME" ]] && {
read A B C D N < <(get-default-ip.sh -m|sed 's;[./]; ;g')
NAS_SERVER_NAME=ontap2-$(printf %02x%02x $C $D)
}
NAS_SERVER_FQDN=$NAS_SERVER_NAME
CIFS_WORKGROUP=${CIFS_WORKGROUP:-FSQE}
LOCAL_USER=root
LOCAL_USER_PASSWD=Sesame~0pen
cifsOption="-workgroup $CIFS_WORKGROUP"
AD_DOMAIN=${AD_NAME#*.}
[[ -n "$AD_DOMAIN" ]] && {
cifsOption="-domain $AD_DOMAIN"
CIFS_WORKGROUP=
NAS_SERVER_FQDN+=.$AD_DOMAIN
}
AD_REALM=$(echo ${AD_DOMAIN} | tr [:lower:] [:upper:])
AD_ADMIN=${AD_ADMIN:-administrator}
AD_PASSWD=${AD_PASSWD:-fsqe2015!}
CIFSVOL1=cifsvol1
CIFSVOL1_AGGR=aggr1_1
CIFSVOL1_SIZE=60G
CIFSVOL1_PATH=/cifs1
SHARENAME1=cifs1
CIFSVOL2=cifsvol2
CIFSVOL2_AGGR=aggr2_1
CIFSVOL2_SIZE=60G
CIFSVOL2_PATH=/cifs2
SHARENAME2=cifs2
#ref1: https://library.netapp.com/ecmdocs/ECMP1366832/html/vserver/export-policy/create.html
#ref2: https://library.netapp.com/ecmdocs/ECMP1366832/html/vserver/export-policy/rule/create.html
#ref3: https://tcler.github.io/2017/08/24/NetApp-pnfs-mds-ds-config
port-available.sh $cluster_managementif_addr 22 --wait
expect -c "spawn ssh admin@$cluster_managementif_addr
set timeout 120
expect {Password:} {
send \"${password}\\r\"
}
expect {${cluster_name}::>} {
send \"vserver create -vserver $VS -subtype default -rootvolume ${VS}_root -rootvolume-security-style mixed -language C.UTF-8 -snapshot-policy default -data-services data-iscsi,data-nfs,data-cifs,data-flexcache -foreground true -aggregate $VS_AGGR\\r\"
expect {Vserver creation completed} {send_user {Vserver creation completed}}
}
expect {${cluster_name}::>} {
send \"vserver export-policy create -vserver $VS -policyname $PolicyName\\r\"
}
expect {${cluster_name}::>} {
send \"vserver export-policy rule create -vserver $VS -policyname $PolicyName -protocol cifs,nfs,nfs3,nfs4,flexcache -clientmatch 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 -rorule any -rwrule any -anon 65534 -allow-suid true -allow-dev true\\r\"
}
expect {${cluster_name}::>} {
send \"volume modify -vserver $VS -volume ${VS}_root -policy $PolicyName -group 0 -user 0\\r\"
}
expect {${cluster_name}::>} {
send \"volume create -volume $VOL1 -aggregate $VOL1_AGGR -size $VOL1_SIZE -state online -unix-permissions ---rwxrwxrwx -type RW -snapshot-policy default -foreground true -tiering-policy none -vserver $VS -junction-path $VOL1_JUNCTION_PATH -policy $PolicyName -group 0 -user 0\\r\"
}
expect {${cluster_name}::>} {
send \"network port broadcast-domain show -ports ${LIF1_0_NODE}:${LIF1_0_PORT} -fields failover-groups\\r\"
expect -re {.*\s+(\S+)\s+${cluster_name}::>} {