Reduxio Solution for Oracle Database 11g · Connected to: Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production With the Partitioning, OLAP, Data Mining and

# yum -y install iscsi-initiator-utils device-mapper device-mapper-event-libs device-mapper-multipath lsscsi sg3_utils

/etc/iscsi/iscsid.conf

node.startup = automatic node.session.timeo.replacement_timeout = 10 node.conn[0].timeo.login_timeout = 15 node.conn[0].timeo.logout_timeout = 15 node.conn[0].timeo.noop_out_interval = 5 node.conn[0].timeo.noop_out_timeout = 5 node.session.initial_login_retry_max 8 node.session.err_timeo.lu_reset_timeout = 300 node.session.err_timeo.tgt_reset_timeout = 300

/etc/udev/rules.d/99-reduxio.rules

SUBSYSTEM=="block" , ACTION=="change", ATTRS{model}=="TCAS", ATTRS{vendor}=="REDUXIO", RUN+="/bin/sh -c '/usr/sbin/iscsiadm -m session -R '" SUBSYSTEM=="block" , ACTION=="change", ATTRS{model}=="TCAS", ATTRS{vendor}=="REDUXIO", ATTR{size}=="0", RUN+="/bin/sh -c 'echo 1 > /sys$DEVPATH/../../delete '"

/etc/multipath.conf

devices { device { vendor "REDUXIO" product "TCAS" revision "2300" path_grouping_policy "group_by_prio" path_checker "tur" hardware_handler "1 alua" prio "alua" failback "immediate" features "0" rr_weight "uniform" path_selector "round-robin 0" no_path_retry "72" rr_min_io_rq 10 rr_min_io 10 user_friendly_names "yes" fast_io_fail_tmo "10" # wait up to 10 seconds before failing io. } } blacklist { # Note: it is highly recommended to blacklist by wwid or vendor instead of device name devnode "^sd[a]$" }

# chkconfig iscsi on ; service iscsi start Starting iscsi daemon: [ OK ] # chkconfig multipathd on ; service multipathd start Starting multipathd daemon: [ OK ] # chkconfig --list iscsi iscsi 0:off 1:off 2:on 3:on 4:on 5:on 6:off # chkconfig --list multipathd multipathd 0:off 1:off 2:on 3:on 4:on 5:on 6:off # udevadm control --reload-rules

# chkconfig iscsi on ; systemctl start iscsi.service Starting iscsi daemon: [ OK ] # chkconfig multipathd on ; systemctl start multipathd Starting multipathd daemon: [ OK ] # chkconfig --list iscsi iscsi 0:off 1:off 2:on 3:on 4:on 5:on 6:off # chkconfig --list multipathd multipathd 0:off 1:off 2:on 3:on 4:on 5:on 6:off # udevadm control --reload-rules

# sudo cat /etc/iscsi/initiatorname.iscsi

$ORA_GRID_HOME/bin/crsctl get css disktimeout 200 $ORA_GRID_HOME/bin/crsctl get css misscount Configuration parameter misscount is not defined.

$ORA_GRID_HOME/bin/crsctl set css misscount 120

$ORA_GRID_HOME/bini/crs_stop -all $ORA_GRID_HOME/bini/crs_start -all

devices { device { vendor "REDUXIO" product "TCAS" revision "2300" path_grouping_policy "group_by_prio" path_checker "tur" hardware_handler "1 alua" prio "alua" failback "immediate" features "0" rr_weight "priorities" no_path_retry 72 rr_min_io_rq 100 user_friendly_names "yes" fast_io_fail_tmo "10" # wait up to 360seconds before failing io. dev_loss_tmo "360" } } defaults {

queue_without_daemon "no" } blacklist { # Note: it is highly recommended to blacklist by wwid or vendor instead of device name devnode "^sd[a]$" }

rdxadmin@reduxio:/ > volumes create –name oradb1 –blocksize 4096 -size 100

[root@mktgdb1 ~]# cat /etc/iscsi/initiatorname.iscsi InitiatorName=iqn.1994-05.com.redhat:aenepheengua

rdxadmin@reduxio:/ > hosts create –name mktgdb1 –iscsi-name iqn.1994-05.com.redhat:aenepheengua

rdxadmin@reduxio:/ > volumes assign oradb1 –host mktgdb1 –lun 1

# iscsiadm --mode discovery -t st -p 10.46.216.11

10.46.216.11:3260,1 iqn.2013-12.com.reduxio: ff4032ff0032000e

$ sudo iscsiadm -m node -l Logging in to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.46.216.11,3260] (multiple) Logging in to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.46.216.22,3260] (multiple) Logging in to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.64.216.11,3260] (multiple) Logging in to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.64.216.22,3260] (multiple) Login to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.46.216.11,3260] successful. Login to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.46.216.22,3260] successful. Login to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.64.216.11,3260] successful. Login to [iface: default, target: iqn.2013-12.com.reduxio:af4032ff0032000e, portal: 10.64.216.22,3260] successful.

# iscsiadm -m node -T iqn.2013-12.com.reduxio:ff4032ff0032000e -p 10.46.216.11 -o update -n node.startup -v automatic

# multipathd show config

# iscsiadm -m session tcp: [1] 10.46.216.11:3260,1 iqn.2013-12.com.reduxio:af4032ff0032000e (non-flash) tcp: [2] 10.46.216.22:3260,1 iqn.2013-12.com.reduxio:af4032ff0032000e (non-flash) tcp: [3] 10.64.216.11:3260,1 iqn.2013-12.com.reduxio:af4032ff0032000e (non-flash) tcp: [4] 10.64.216.22:3260,1 iqn.2013-12.com.reduxio:af4032ff0032000e (non-flash)

# multipath –ll mpatha (36f4032f00035000000000000000008ab) dm-3 REDUXIO,TCAS size=500G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw |-+- policy='round-robin 0' prio=130 status=active | `- 10:0:0:120 sde 8:64 active ready running `-+- policy='round-robin 0' prio=10 status=enabled |- 4:0:0:120 sdd 8:48 active ready running `- 3:0:0:120 sdc 8:32 active ready running

# mkfs.ext4 /dev/mapper/mpatha

# mkdir -p /oracle_data/oradata # echo "/dev/disk-byuuid/ 93845f7g-263c-4903-8cc6-db26a2b3579b /oracle_data ext4 _netdev 0 0" >> /etc/fstab # chown -R oracle:oinstall /oracle_data

# mount /oracle_data # df /oracle_data Filesystem 1K-blocks Used Available Use% Mounted on /dev/mapper/mpatha 103081248 61044 97777324 1% /oracle_data

# vi ~oracle/.bash_profile

export ORACLE_HOME=/u01/app/oracle/11.2/db_1/ export ORACLE_INVENTORY=/home/oracle/oraInventory export ORACLE_BASE=/u01/app/oracle export ORACLE_INVENTORY=/home/oracle/oraInventory export ORACLE_BASE=/u01/app/oracle export ORACLE_SID=TEST export ORACLE_DB_HOME=/u01/app/oracle/product/11.2.0/db_1 export SYS_PASSOWRD=manager export SYSTEM_PASSWORD=manager export ASM_PASSWORD=manager export ORACLE_DATA_DIR=/oracle_data/oradata export ORACLE_ARCHIVE_DIR=/oracle_data/archives PATH=$PATH:$ORACLE_BASE/11.2/db_1/bin:$HOME/bin export PATH

# su – oracle

[oracle@mktgdb1 ~]$ echo > /etc/oratab [oracle@mktgdb1 ~]$ dbca -silent -createDatabase -templateName General_Purpose.dbc -gdbname $ORACLE_SID -sid $ORACLE_SID -responseFile NO_VALUE -characterSet UTF8 -memoryPercentage 30 -SYSPASSWORD $SYS_PASSOWRD -SYSTEMPASSWORD $SYSTEM_PASSWORD -datafileDestination $ORACLE_DATA_DIR -recoveryAreaDestination $ORACLE_DATA_DIR -redoLogFileSize 5000 Copying database files 1% complete 3% complete 10% complete 17% complete 24% complete 35% complete Creating and starting Oracle instance 37% complete 42% complete 47% complete 52% complete 53% complete 56% complete 58% complete Registering database with Oracle Restart 64% complete Completing Database Creation 68% complete 71% complete 75% complete 85% complete 96% complete 100% complete Look at the log file "/u01/app/oracle/cfgtoollogs/dbca/TEST/TEST.log" for further details.

[oracle@mktgdb1 ~]$ ls -lah /oracle_data/oradata/TEST total 16G drwxr-x--- 3 oracle oinstall 4.0K Dec 14 12:19 . drwxr-xr-x 5 oracle oinstall 4.0K Dec 14 12:04 .. -rw-r----- 1 oracle asmadmin 9.3M Dec 14 12:21 control01.ctl -rw-r----- 1 oracle asmadmin 9.3M Dec 14 12:21 control02.ctl drwxr-x--- 2 oracle asmadmin 4.0K Dec 14 12:18 onlinelog -rw-r----- 1 oracle asmadmin 4.9G Dec 14 12:20 redo01.log -rw-r----- 1 oracle asmadmin 4.9G Dec 14 12:20 redo02.log -rw-r----- 1 oracle asmadmin 4.9G Dec 14 12:20 redo03.log -rw-r----- 1 oracle asmadmin 491M Dec 14 12:20 sysaux01.dbf -rw-r----- 1 oracle asmadmin 741M Dec 14 12:20 system01.dbf -rw-r----- 1 oracle asmadmin 30M Dec 14 12:19 temp01.dbf -rw-r----- 1 oracle asmadmin 71M Dec 14 12:20 undotbs01.dbf -rw-r----- 1 oracle asmadmin 5.1M Dec 14 12:20 users01.dbf

[oracle@mktgdb1 ~]$ mkdir -p $ORACLE_ARCHIVE_DIR [oracle@mktgdb1 ~]$ sqlplus -L / as sysdba SQL*Plus: Release 11.2.0.4.0 Production on Wed Dec 14 13:14:09 2016 Copyright (c) 1982, 2013, Oracle. All rights reserved.

Connected to: Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production With the Partitioning, OLAP, Data Mining and Real Application Testing options SQL> alter system set log_archive_dest_1='LOCATION=$ORACLE_ARCHIVE_DIR' scope=both; System altered. SQL> alter system set "_bump_highwater_mark_count"=50 scope=spfile; System altered. SQL> alter system set filesystemio_options = 'setall' scope=spfile; System altered. SQL> shutdown immediate; Database closed. Database dismounted. ORACLE instance shut down. SQL> startup mount; ORACLE instance started. Total System Global Area 1.5132E+10 bytes Fixed Size 2267992 bytes Variable Size 2181039272 bytes Database Buffers 1.2918E+10 bytes Redo Buffers 30527488 bytes Database mounted SQL> alter database archivelog; SQL> alter database open;

$ORACLE_BASE/diag/rdbms/$ORACLE_SID/$ORACLE_SID/trace/alert_$ORACLE_SID.log

/u01/app/oracle/diag/rdbms/test/TEST/trace/alert_test.log

rdxadmin@reduxio:/ ➜ volumes clone oradb1 -name oradb1C1

-timestamp 02/17/2016-11:39:00 Clone created successfully

[oracle@mktgdb1 ~]$ sqlplus -L / as sysdba SQL*Plus: Release 11.2.0.4.0 Production on Wed Dec 14 13:14:09 2016 Copyright (c) 1982, 2013, Oracle. All rights reserved. Connected to: Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production With the Partitioning, OLAP, Data Mining and Real Application Testing options SQL> shutdown immediate; Database closed. Database dismounted. ORACLE instance shut down. SQL>

[oracle@mktgdb1]$ sudo umount /oracle_data

rdxadmin@reduxio:/ > volumes unassign oradb1 -host mktgdb1 Successfully deleted: dbvol1<->mktgdb1

rdxadmin@reduxio:/ > volumes assign oradb1C1 -host mktgdb1 Volume assigned successfully

[oracle@mktgdb1]$ sudo iscsiadm -m node -R Rescanning session [sid: 1, target: iqn.2013-12.com.reduxio:af4032f0003a000e, portal: 10.64.216.11,3260] Rescanning session [sid: 2, target: iqn.2013-12.com.reduxio:af4032f0003a000e, portal: 10.46.216.11,3260] Rescanning session [sid: 3, target: iqn.2013-12.com.reduxio:af4032f0003a000e, portal: 10.64.216.22,3260] Rescanning session [sid: 4, target: iqn.2013-12.com.reduxio:af4032f0003a000e, portal: 10.46.216.22,3260]

[oracle@mktgdb1]$ sudo multipath -r create: mpathdb (36f4032f0003a0000000000000000002a) undef REDUXIO,TCAS size=100G features='0' hwhandler='1 alua' wp=undef |-+- policy='round-robin 0' prio=50 status=undef | |- 33:0:0:1 sdc 8:32 active ready running | `- 34:0:0:1 sdd 8:48 active ready running `-+- policy='round-robin 0' prio=10 status=undef |- 35:0:0:1 sde 8:64 active ready running `- 36:0:0:1 sdf 8:80 active ready running

[oracle@mktgdb1]$ sudo multipath -ll mpathdb (36f4032f0003a0000000000000000002a) dm-4 REDUXIO,TCAS size=100G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw |-+- policy='round-robin 0' prio=50 status=active | |- 33:0:0:1 sdc 8:32 active ready running | `- 34:0:0:1 sdd 8:48 active ready running `-+- policy='round-robin 0' prio=10 status=enabled |- 35:0:0:1 sde 8:64 active ready running `- 36:0:0:1 sdf 8:80 active ready running

/dev/disk/by-uuid/ed931b4e-263c-4903-8cc6-db26a2b3282a /oracle_data ext4 _netdev 0 0

[oracle@mktgdb1]$ sudo mount /oracle_data

[oracle@mktgdb1]$ sqlplus -L / as sysdba SQL*Plus: Release 11.2.0.4.0 Production on Sun Dec 25 15:15:19 2016 Copyright (c) 1982, 2013, Oracle. All rights reserved. Connected to an idle instance. SQL> startup mount; ORACLE instance started. Total System Global Area 1.5132E+10 bytes Fixed Size 2267992 bytes Variable Size 2181039272 bytes Database Buffers 1.2918E+10 bytes Redo Buffers 30527488 bytes Database mounted. SQL>

Sun Dec 25 15:15:31 2016 ALTER DATABASE MOUNT Successful mount of redo thread 1, with mount id 2229721747 Database mounted in Exclusive Mode Lost write protection disabled Completed: ALTER DATABASE MOUNT Sun Dec 25 15:22:44 2016 alter database open Beginning crash recovery of 1 threads

parallel recovery started with 15 processes Started redo scan Completed redo scan read 190125 KB redo, 10986 data blocks need recovery Started redo application at Thread 1: logseq 2, block 1393 Recovery of Online Redo Log: Thread 1 Group 2 Seq 2 Reading mem 0 Mem# 0: /oracle_data/oradata/TEST/redo02.log Completed redo application of 163.32MB Completed crash recovery at Thread 1: logseq 2, block 381644, scn 1156259 10986 data blocks read, 10986 data blocks written, 190125 redo k-bytes read Sun Dec 25 15:22:53 2016 Thread 1 advanced to log sequence 3 (thread open) Thread 1 opened at log sequence 3 Current log# 3 seq# 3 mem# 0: /oracle_data/oradata/TEST/redo03.log Successful open of redo thread 1 MTTR advisory is disabled because FAST_START_MTTR_TARGET is not set Sun Dec 25 15:22:54 2016 SMON: enabling cache recovery [24076] Successfully onlined Undo Tablespace 2. Undo initialization finished serial:0 start:961359524 end:961359664 diff:140 (1 seconds) Verifying file header compatibility for 11g tablespace encryption.. Verifying 11g file header compatibility for tablespace encryption completed SMON: enabling tx recovery Database Characterset is UTF8 No Resource Manager plan active Sun Dec 25 15:22:54 2016 replication_dependency_tracking turned off (no async multimaster replication found) Starting background process QMNC Sun Dec 25 15:22:55 2016 QMNC started with pid=53, OS id=24198 Completed: alter database open Sun Dec 25 15:22:56 2016 db_recovery_file_dest_size of 48732 MB is 0.00% used. This is a user-specified limit on the amount of space that will be used by this database for recovery-related files, and does not reflect the amount of space available in the underlying filesystem or ASM diskgroup. Sun Dec 25 15:22:57 2016 Starting background process CJQ0 Sun Dec 25 15:22:57 2016 CJQ0 started with pid=56, OS id=24221

[oracle@mktgdb1 ~]$ sqlplus -L / as sysdba SQL*Plus: Release 11.2.0.4.0 Production on Wed Dec 14 13:14:09 2016 Copyright (c) 1982, 2013, Oracle. All rights reserved.

Connected to: Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production With the Partitioning, OLAP, Data Mining and Real Application Testing options SQL> shutdown immediate; Database closed. Database dismounted. ORACLE instance shut down. SQL>

[oracle@mktgdb1]$ sudo umount /oracle_data

rdxadmin@reduxio:/ > volumes unassign oradb1 -host mktgdb1 Successfully deleted: dbvol1<->mktgdb1

rdxadmin@reduxio:/ ➜ volumes revert oradb1 -timestamp

02/17/2016-11:39:00 –force Volume oradb1 reverted

[oracle@mktgdb1]$ sudo mount /oracle_data

[oracle@mktgdb1]$ sqlplus -L / as sysdba SQL*Plus: Release 11.2.0.4.0 Production on Sun Dec 25 15:15:19 2016 Copyright (c) 1982, 2013, Oracle. All rights reserved. Connected to an idle instance. SQL> startup mount; ORACLE instance started. Total System Global Area 1.5132E+10 bytes Fixed Size 2267992 bytes Variable Size 2181039272 bytes Database Buffers 1.2918E+10 bytes

Redo Buffers 30527488 bytes Database mounted. SQL>

Sun Dec 25 15:15:31 2016 ALTER DATABASE MOUNT Successful mount of redo thread 1, with mount id 2229721747 Database mounted in Exclusive Mode Lost write protection disabled Completed: ALTER DATABASE MOUNT Sun Dec 25 15:22:44 2016 alter database open Beginning crash recovery of 1 threads parallel recovery started with 15 processes Started redo scan Completed redo scan read 190125 KB redo, 10986 data blocks need recovery Started redo application at Thread 1: logseq 2, block 1393 Recovery of Online Redo Log: Thread 1 Group 2 Seq 2 Reading mem 0 Mem# 0: /oracle_data/oradata/TEST/redo02.log Completed redo application of 163.32MB Completed crash recovery at Thread 1: logseq 2, block 381644, scn 1156259 10986 data blocks read, 10986 data blocks written, 190125 redo k-bytes read Sun Dec 25 15:22:53 2016 Thread 1 advanced to log sequence 3 (thread open) Thread 1 opened at log sequence 3 Current log# 3 seq# 3 mem# 0: /oracle_data/oradata/TEST/redo03.log Successful open of redo thread 1 MTTR advisory is disabled because FAST_START_MTTR_TARGET is not set Sun Dec 25 15:22:54 2016 SMON: enabling cache recovery [24076] Successfully onlined Undo Tablespace 2. Undo initialization finished serial:0 start:961359524 end:961359664 diff:140 (1 seconds) Verifying file header compatibility for 11g tablespace encryption.. Verifying 11g file header compatibility for tablespace encryption completed SMON: enabling tx recovery Database Characterset is UTF8 No Resource Manager plan active Sun Dec 25 15:22:54 2016 replication_dependency_tracking turned off (no async multimaster replication found) Starting background process QMNC Sun Dec 25 15:22:55 2016 QMNC started with pid=53, OS id=24198 Completed: alter database open Sun Dec 25 15:22:56 2016 db_recovery_file_dest_size of 48732 MB is 0.00% used. This is a user-specified limit on the amount of space that will be used by this database for recovery-related files, and does not reflect the amount of space available in the underlying filesystem or ASM diskgroup. Sun Dec 25 15:22:57 2016 Starting background process CJQ0

Sun Dec 25 15:22:57 2016 CJQ0 started with pid=56, OS id=24221