Initial robustification of HADB tests

dev/com.ibm.rls.jdbc/src/com/ibm/rls/jdbc/LogAccessTagger.java

@@ -0,0 +1,17 @@
+/*******************************************************************************
+ * Copyright (c) 2024 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package com.ibm.rls.jdbc;
+
+/**
+ * Switch on
+ */
+public class LogAccessTagger {
+
+}

dev/com.ibm.rls.jdbc/src/com/ibm/ws/recoverylog/custom/jdbc/impl/SQLMultiScopeRecoveryLog.java

@@ -312,6 +312,9 @@ public class SQLMultiScopeRecoveryLog implements LogCursorCallback, MultiScopeLo
     private int _throttleWaiters;
     private final static int _waiterThreshold;
 
+    // Switch on SQL comments for testing
+    private static Boolean _tagSQL;
+
     static {
         int waiterThreshold = 6;
         try {
@@ -331,6 +334,17 @@ public Integer run() {
         _waiterThreshold = waiterThreshold;
         if (tc.isDebugEnabled())
             Tr.debug(tc, "Throttle waiter threshold set to ", _waiterThreshold);
+
+        try {
+            _tagSQL = AccessController.doPrivileged(
+                                                    new PrivilegedExceptionAction<Boolean>() {
+                                                        @Override
+                                                        public Boolean run() {
+                                                            return Boolean.getBoolean("com.ibm.ws.recoverylog.custom.jdbc.tagSQL");
+                                                        }
+                                                    });
+        } catch (PrivilegedActionException e) {
+        }
     }
 
     private boolean _throttleEnabled;
@@ -2709,7 +2723,7 @@ private void assertLogOwnershipAtOpenWithLatching(Connection conn) throws SQLExc
      */
     private void assertLogOwnershipAtOpenPeerLocking(Connection conn) throws SQLException, InternalLogException {
         if (tc.isEntryEnabled())
-            Tr.entry(tc, "assertLogOwnershipAtOpenPeerLocking", new java.lang.Object[] { conn, this });
+            Tr.entry(tc, "assertLogOwnershipAtOpenPeerLocking", conn, this);
 
         boolean takeLock = false;
         Statement readForUpdateStmt = null;
@@ -2718,7 +2732,7 @@ private void assertLogOwnershipAtOpenPeerLocking(Connection conn) throws SQLExce
 
         try {
             readForUpdateStmt = conn.createStatement();
-            readForUpdateRS = readHADBLock(readForUpdateStmt, _logIdentifierString);
+            readForUpdateRS = readHADBLock(readForUpdateStmt, _logIdentifierString, "--assertLogOwnershipAtOpenPeerLocking");
             if (readForUpdateRS.next()) {
                 // We found the HA Lock row
                 String storedServerName = readForUpdateRS.getString(1);
@@ -2967,10 +2981,8 @@ private void insertLockingRow(Connection conn, String fullTableName, short servi
                               fullTableName +
                               " (SERVER_NAME, SERVICE_ID, RU_ID, RUSECTION_ID, RUSECTION_DATA_INDEX, DATA)" +
                               " VALUES (?,?,?,?,?,?)";
-        if (tc.isDebugEnabled())
-            Tr.debug(tc, "Insert LOCKING row using - " + insertString);
 
-        int ret;
+        final int ret;
         try (PreparedStatement specStatement = conn.prepareStatement(insertString)) {
             specStatement.setString(1, _currentProcessServerName);
             specStatement.setShort(2, serviceId);
@@ -2982,10 +2994,8 @@ private void insertLockingRow(Connection conn, String fullTableName, short servi
             ret = specStatement.executeUpdate();
         }
 
-        if (tc.isDebugEnabled())
-            Tr.debug(tc, "Have inserted HA LOCKING ROW with return: " + ret);
         if (tc.isEntryEnabled())
-            Tr.exit(tc, "insertLockingRow");
+            Tr.exit(tc, "insertLockingRow", ret);
     }
 
     //------------------------------------------------------------------------------
@@ -3683,12 +3693,18 @@ public void closeConnectionAfterBatch(Connection conn, int initialIsolation) thr
 
     // helper method
     private ResultSet readHADBLock(Statement lockingStmt, String logIdentifierString) throws SQLException {
+        return readHADBLock(lockingStmt, logIdentifierString, "");
+    }
+
+    // helper method
+    private ResultSet readHADBLock(Statement lockingStmt, String logIdentifierString, String sqltag) throws SQLException {
         // Use RDBMS SELECT FOR UPDATE to lock table for recovery
         String queryString = "SELECT SERVER_NAME, RUSECTION_ID" +
                              " FROM " + _recoveryTableName + logIdentifierString + _recoveryTableNameSuffix +
                              (DBProduct.Sqlserver == dbProduct ? " WITH (ROWLOCK, UPDLOCK, HOLDLOCK)" : "") +
                              " WHERE RU_ID=-1" +
-                             (DBProduct.Sqlserver == dbProduct ? "" : " FOR UPDATE");
+                             ((DBProduct.Sqlserver == dbProduct ? "" : " FOR UPDATE") +
+                              (_tagSQL ? sqltag : ""));
         if (tc.isDebugEnabled())
             Tr.debug(tc, "Attempt to select the HA LOCKING ROW for UPDATE - " + queryString);
         return lockingStmt.executeQuery(queryString);
@@ -3955,7 +3971,7 @@ private void assertDBTableExists(Connection conn, String logIdentifierString) th
             try {
                 touchStmt = conn.createStatement();
                 // This is just a touch test to see if we need to create the table (surely we could use  DatabaseMetaData.getTables)
-                touchRS = readHADBLock(touchStmt, logIdentifierString);
+                touchRS = readHADBLock(touchStmt, logIdentifierString, "--assertDBTableExists");
 
                 if (touchRS != null)
                     try {
@@ -4446,7 +4462,8 @@ boolean internalClaimRecoveryLogs(Connection conn, boolean isHomeServer) throws
                              " FROM " + _recoveryTableName + "PARTNER_LOG" + _recoveryTableNameSuffix +
                              (DBProduct.Sqlserver == dbProduct ? " WITH (ROWLOCK, UPDLOCK, HOLDLOCK)" : "") +
                              " WHERE RU_ID=-1" +
-                             (DBProduct.Sqlserver == dbProduct ? "" : " FOR UPDATE");
+                             ((DBProduct.Sqlserver == dbProduct ? "" : " FOR UPDATE") +
+                              (_tagSQL ? "--internalClaimRecoveryLogs" : ""));
         if (tc.isDebugEnabled())
             Tr.debug(tc, "Attempt to select the HA LOCKING ROW for UPDATE using - " + queryString);
 

dev/com.ibm.ws.transaction.hadb_fat.derby.1/fat/src/tests/FailoverTest.java

@@ -129,5 +129,6 @@ protected static void commonCleanup(String testClassName) throws Exception {
 
             break;
         }
+        TxTestContainerSuite.dropTables("WAS_TRAN_LOG", "WAS_PARTNER_LOG");
     }
 }

dev/com.ibm.ws.transaction.hadb_fat.derby.1/fat/src/tests/FailoverTest1.java

@@ -21,7 +21,6 @@
 import java.util.Collections;
 import java.util.List;
 
-import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -39,7 +38,6 @@
 import componenttest.annotation.SkipIfSysProp;
 import componenttest.custom.junit.runner.FATRunner;
 import componenttest.topology.impl.LibertyServer;
-import suite.FATSuite;
 
 /**
  * These tests are designed to exercise the ability of the SQLMultiScopeRecoveryLog (transaction logs stored
@@ -137,11 +135,6 @@ public class FailoverTest1 extends FailoverTest {
                                                         "com.ibm.ws.transaction_multipleretries",
     };
 
-    @AfterClass
-    public static void afterSuite() {
-        FATSuite.afterSuite("WAS_TRAN_LOG", "WAS_PARTNER_LOG");
-    }
-
     @BeforeClass
     public static void setUp() throws Exception {
         FailoverTest.commonSetUp(FailoverTest1.class);
@@ -189,7 +182,7 @@ public void testHADBRecoverableRuntimeFailover() throws Exception {
 
         server = defaultServer;
 
-        HADBTestControl.write(HADBTestType.RUNTIME, -4498, 12, 1);
+        HADBTestControl.write(HADBTestType.RUNTIME, -4498, 14, 1);
 
         FATUtils.startServers(runner, server);
 
@@ -213,7 +206,7 @@ public void testHADBRecoverableFailureMultipleRetries() throws Exception {
 
         server = retriesServer;
 
-        HADBTestControl.write(HADBTestType.RUNTIME, -4498, 12, 5); // Can fail up to 5 times
+        HADBTestControl.write(HADBTestType.RUNTIME, -4498, 11, 5); // Can fail up to 5 times
 
         FATUtils.startServers(runner, server);
 
@@ -224,19 +217,6 @@ public void testHADBRecoverableFailureMultipleRetries() throws Exception {
         // Should see a message like
         // WTRN0100E: Cannot recover from SQLException when forcing SQL RecoveryLog tranlog for server com.ibm.ws.transaction
         assertNotNull("No error message signifying log failure", server.waitForStringInLog("Cannot recover from SQLException when forcing SQL RecoveryLog"));
-
-        // We need to tidy up the environment at this point. We cannot guarantee
-        // test order, so we should ensure
-        // that we do any necessary recovery at this point
-        FATUtils.stopServers(server);
-
-        FATUtils.startServers(runner, retriesServer);
-
-        // RTC defect 170741
-        // Wait for recovery to be driven - this may suffer from a delay (see
-        // RTC 169082), so wait until the "recover("
-        // string appears in the messages.log
-        assertNotNull("Recovery didn't happen for " + server.getServerName(), server.waitForStringInTrace("Performed recovery for " + server.getServerName()));
     }
 
     /**
@@ -289,7 +269,7 @@ public void testHADBRecoverableStartupFailover() throws Exception {
 
         server = defaultServer;
 
-        HADBTestControl.write(HADBTestType.STARTUP, -4498, 11, 1);
+        HADBTestControl.write(HADBTestType.STARTUP, -4498, 11, 1, "--assertLogOwnershipAtOpenPeerLocking");
 
         FATUtils.startServers(runner, server);
 
@@ -304,7 +284,7 @@ public void testHADBNonRecoverableStartupFailover() throws Exception {
         server = defaultServer;
         serverMsgs = new String[] { "WTRN0107W", "WTRN0000E", "WTRN0112E", "WTRN0153W" };
 
-        HADBTestControl.write(HADBTestType.STARTUP, -3, 11, 1);
+        HADBTestControl.write(HADBTestType.STARTUP, -3, 11, 1, "--assertLogOwnershipAtOpenPeerLocking");
 
         FATUtils.startServers(runner, server);
         StringBuilder sb = runInServlet(server, SERVLET_NAME, "driveTransactions");
@@ -320,7 +300,7 @@ public void testHADBEarlyNonRecoverableStartupFailover() throws Exception {
         server = defaultServer;
         serverMsgs = new String[] { "WTRN0107W", "WTRN0000E", "WTRN0112E", "WTRN0153W" };
 
-        HADBTestControl.write(HADBTestType.STARTUP, -3, 1, 1);
+        HADBTestControl.write(HADBTestType.STARTUP, -3, 4, 1, "--internalClaimRecoveryLogs");
 
         FATUtils.startServers(runner, server);
         StringBuilder sb = runInServlet(server, SERVLET_NAME, "driveTransactions");

dev/com.ibm.ws.transaction.hadb_fat.derby.1/fat/src/tests/FailoverTest2.java

@@ -16,7 +16,6 @@
 
 import java.util.Collections;
 
-import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -34,7 +33,6 @@
 import componenttest.annotation.SkipIfSysProp;
 import componenttest.custom.junit.runner.FATRunner;
 import componenttest.topology.impl.LibertyServer;
-import suite.FATSuite;
 
 /**
  * These tests are designed to exercise the ability of the SQLMultiScopeRecoveryLog (transaction logs stored
@@ -130,11 +128,6 @@ public class FailoverTest2 extends FailoverTest {
                                                         "com.ibm.ws.transaction_recover",
     };
 
-    @AfterClass
-    public static void afterSuite() {
-        FATSuite.afterSuite("WAS_TRAN_LOG", "WAS_PARTNER_LOG");
-    }
-
     @BeforeClass
     public static void setUp() throws Exception {
         FailoverTest.commonSetUp(FailoverTest2.class);

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_ANYDBCLOUD001.fastcheck/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_ANYDBCLOUD001.longleasecompete/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_ANYDBCLOUD002.fastcheck/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_multipleretries/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_nonretriable/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_recover/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_retriable/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_retriablecloud/jvm.options

@@ -0,0 +1 @@
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/publish/servers/com.ibm.ws.transaction_stalecloud/jvm.options

@@ -1 +1,2 @@
--Dcom.ibm.ws.recoverylog.disablehomelogdeletion=true
+-Dcom.ibm.ws.recoverylog.disablehomelogdeletion=true
+-Dcom.ibm.ws.recoverylog.custom.jdbc.tagSQL=true

dev/com.ibm.ws.transaction.hadb_fat.derby.1/test-bundles/ifxlib/src/com/informix/jdbcx/IfxConnection.java

@@ -31,6 +31,7 @@
 import java.util.Properties;
 import java.util.concurrent.Executor;
 
+import com.ibm.tx.jta.ut.util.HADBTestControl;
 import com.informix.database.ConnectionManager;
 
 /**
@@ -65,6 +66,8 @@ public class IfxConnection implements Connection {
     private static boolean _testingLeaselogClaimFlag;
     private static boolean _testingLeaselogGetFlag;
 
+    private static HADBTestControl _control;
+
     IfxConnection(Connection realConn) {
         System.out.println("IfxConnection(" + wrappedConn + "): construct wrapped connection using - " + realConn);
         wrappedConn = realConn;
@@ -907,6 +910,10 @@ public static int getSimSQLCode() {
         return simSQLCode;
     }
 
+    public static HADBTestControl getTestControl() {
+        return _control;
+    }
+
     /**
      * @param simSQLCode
      *            the simSQLCode to set
@@ -1033,4 +1040,15 @@ public int getNetworkTimeout() throws SQLException {
         return 0;
     }
 
+    /**
+     * @param testControl
+     */
+    public static void setControl(HADBTestControl testControl) {
+        _control = testControl;
+    }
+
+    public static HADBTestControl getControl() {
+        return _control;
+    }
+
 }

dev/com.ibm.ws.transaction.hadb_fat.derby.1/test-bundles/ifxlib/src/com/informix/jdbcx/IfxPooledConnection.java

@@ -108,6 +108,8 @@ else if (unwrappedConnection != null)
                 int numberOfFailuresInt = testControl.getNumberOfFailuresInt();
                 System.out.println("SIMHADB: Stored column numberoffailures is: " + numberOfFailuresInt);
 
+                IfxConnection.setControl(testControl);
+
                 switch (testType) {
                     case STARTUP:
                         // We abuse the failovervalInt parameter. If it is set to