FileDocCategorySizeDatePackage
RecoveryManager.javaAPI DocGlassfish v2 API63788Wed Jun 13 23:03:46 BST 2007com.sun.jts.CosTransactions

RecoveryManager

public class RecoveryManager extends Object
This class manages information required for recovery, and also general state regarding transactions in a process.
version
0.01
author
Simon Holdsworth, IBM Corporation
see

Fields Summary
private static Enumeration
uniqueRMSet
list of XA Resources to be recovered.
private static boolean
initialised
This attribute indicates whether initialisation has been started.
private static int
resyncCoords
This attribute indicates the number of Coordinator objects which require resync. This is set to the number of in-doubt transactions recovered from the log, then decreased as transactions are resolved.
private static ResyncThread
resyncThread
This attribute records the thread which is used to perform resync during restart
private static volatile EventSemaphore
resyncInProgress
This attribute is used to block new requests while there are Coordinators which still require resync.
private static volatile EventSemaphore
recoveryInProgress
This attribute is used to block requests against RecoveryCoordinators or CoordinatorResources before recovery has completed.
private static volatile EventSemaphore
uniqueRMSetReady
This attribute is used by the Recovery Thread to know if the xaResource list is ready in case manual recovery is attempted.
private static Hashtable
coordsByGlobalTID
private static Hashtable
coordsByLocalTID
private static Hashtable
transactionIds
Mapping between transactionIds and threads. This is used to ensure there is at most one thread doing work in a transaction.
private static Hashtable
inCompleteTxMap
Mapping between incompleteTxIds and their commit decisions.
private static com.sun.enterprise.autotxrecovery.TransactionRecovery
txRecovery
private static Object
lockObject
This is intented to be used as a lock object.
static Logger
_logger
Constructors Summary
Methods Summary
static booleanaddCoordinator(GlobalTID globalTID, java.lang.Long localTID, CoordinatorImpl coord, int timeout)
Sets up the local and global identifier to Coordinator mapping as given.

If the global identifier has already got associated information, the operation returns false.

The timeout value, if non-zero, is used to establish a time-out for the transaction; if the local identifier to Coordinator association exists after the time-out period, then the TimeoutManager will attempt to roll the transaction back.

param
globalTID The global identifier for the transaction.
param
localTID The local identifier for the transaction.
param
coord The Coordinator for the transaction.
param
timeout The timeout for the transaction.
param
log The log object for the transaction.
return
Indicates success of the operation.
see


        boolean result = true;

        // Attempt to add the global and local indentifier to
        // Coordinator associations to the maps.

        coordsByGlobalTID.put(globalTID,coord);
        coordsByLocalTID.put(localTID,coord);

        // Set up the timeout for the transaction.  When active, the
        // timeout thread will periodically examine the map and abort
        // any active transactions on it that have gone beyond their
        // allocated time.

        if (timeout != 0) {
            TimeoutManager.setTimeout(localTID, TimeoutManager.ACTIVE_TIMEOUT,
                                      timeout);
        }

        return result;
    
static voidaddToIncompleTx(CoordinatorImpl coord, boolean commit)

        inCompleteTxMap.put(coord, new Boolean(commit));
    
static voidcreateRecoveryFile(java.lang.String serverName)

        try {
            int[] result = new int[1];
            String logPath = Configuration.getDirectory(Configuration.LOG_DIRECTORY,
                                                 Configuration.JTS_SUBDIRECTORY,
                                                 result);
            if( result[0] == Configuration.DEFAULT_USED ||
                result[0] == Configuration.DEFAULT_INVALID ) {
                if( result[0] == Configuration.DEFAULT_INVALID ) {
                    logPath = ".";
                }
            }
            File recoveryFile = LogControl.recoveryIdentifierFile(serverName,logPath);
            RandomAccessFile raf = new RandomAccessFile(recoveryFile,"rw");
            raf.writeBytes(serverName);
            raf.setLength(serverName.length());
            raf.close();
        } catch (Exception ex) {
            _logger.log(Level.WARNING,"jts.exception_in_recovery_file_handling",ex);
        }
    
static voiddbXARecovery()

        Enumeration xaResources = RecoveryManager.uniqueRMSet;
        String manualRecovery =
            Configuration.getPropertyValue(Configuration.MANUAL_RECOVERY);
        // if ManualRecovery property is not set, do not attempt XA recovery.
        if (manualRecovery == null  ||
                !(manualRecovery.equalsIgnoreCase("true"/*#Frozen*/))) {
            try {
            resyncComplete(false, false);
            } catch (Throwable ex) { }
            return;
        }

        if (Thread.currentThread().getName().equals("JTS Resync Thread"/*#Frozen*/)) {
            if (uniqueRMSetReady != null) {
                try {
                    uniqueRMSetReady.waitEvent();
                    xaResources = RecoveryManager.uniqueRMSet;
                } catch (InterruptedException exc) {
		    _logger.log(Level.SEVERE,"jts.wait_for_resync_complete_interrupted");
		    String msg = LogFormatter.getLocalizedMessage(_logger,
		              "jts.wait_for_resync_complete_interrupted");
		    throw  new org.omg.CORBA.INTERNAL(msg);
                }
            }
        }

        // sanity check
        if (xaResources == null) {
            try {
            resyncComplete(false, false);
            } catch (Throwable ex) { }
            return;
        }

        // Get global TIDs
        Map gtidMap = LogDBHelper.getInstance().getGlobalTIDMap();

        Set uniqueXids = new HashSet();

        while (xaResources.hasMoreElements()) {

            XAResource xaResource = (XAResource) xaResources.nextElement();

            // Get the list of XIDs which represent in-doubt transactions
            // for the database.

            Xid[] inDoubtXids = getInDoubtXids(xaResource);
            // uniqueXids.clear();
            if (inDoubtXids == null || inDoubtXids.length == 0) {
                    continue; // No in-doubt xids for this resource.
            }
            for (int i = 0; i < inDoubtXids.length; i++) {

                    // check to see if the xid belongs to this server.

                    String branchQualifier =
                        new String(inDoubtXids[i].getBranchQualifier());
                    String serverName = Configuration.getServerName();
                    
                    if (branchQualifier.startsWith(serverName)) {

                        // check if the xid is a duplicate. i.e., Xids
                        // which have same globalId and branchId are
                        // considered duplicates. Note that the
                        // branchId format is (serverId, rmId). This is
                        // to make sure that at most one OTSResource object
                        // is registered with the coordinator per transaction
                        // per RM.
                        
                        String xidStr = stringifyXid(inDoubtXids[i]);
                        if (!uniqueXids.contains(inDoubtXids[i])) { // unique xid
                            if(_logger.isLoggable(Level.FINE))
                            {
                                _logger.logp(Level.FINE,"RecoveryManager",
                                        "dbXARecovery",
                                        " This xid is UNIQUE " +
                                        inDoubtXids[i]);
                            }

                            uniqueXids.add(inDoubtXids[i]); // add to uniqueList

                            try {
                            byte[] gtrid = inDoubtXids[i].getGlobalTransactionId();
                            GlobalTID gtid = GlobalTID.fromTIDBytes(gtrid);
                            Long localTID = (Long)gtidMap.get(gtid);
                            if (localTID == null) {
                                 xaResource.rollback(inDoubtXids[i]);
                            } else {
                                 xaResource.commit(inDoubtXids[i], true);
                                 LogDBHelper.getInstance().deleteRecord(localTID.longValue());
                            }
                            } catch (Exception ex) { ex.printStackTrace(); }
                        } else {
                            if(_logger.isLoggable(Level.FINE))
                            {
                                _logger.logp(Level.FINE,"RecoveryManager",
                                            "dbXARecovery",
                                            " This xid is NOTUNIQUE " +
                                            inDoubtXids[i]);
                            }
                        }
                    } else {
                        if(_logger.isLoggable(Level.FINE))
                        {
                            _logger.logp(Level.FINE,"RecoveryManager",
                                        "dbXARecovery",
                                        " This xid doesn't belong to me " +
                                        inDoubtXids[i]);
                        }

                    }
                }
        }
        try {
        resyncComplete(false, false);
        } catch (Throwable ex) { ex.printStackTrace(); }
    
static CoordinatorImplgetCoordinator(GlobalTID globalTID)
Returns a reference to the Coordinator object that corresponds to the global identifier passed as a parameter.

param
globalTID The global identifier for the transaction.
return
The Coordinator for the transaction.
see


        CoordinatorImpl result = (CoordinatorImpl)
            coordsByGlobalTID.get(globalTID);

        return result;
    
static CoordinatorImpl[]getCoordinators()
Returns an array of Coordinator objects currently active.

param
return
The array of Coordinators.
see


        int size = coordsByGlobalTID.size();
        CoordinatorImpl[] result = new CoordinatorImpl[size];

        Enumeration coords = coordsByGlobalTID.elements();

		for(int pos = 0;pos<size;){
            result[pos++] = (CoordinatorImpl) coords.nextElement();
        }

        return result;
    
static java.util.HashtablegetCoordsByGlobalTID()

		return coordsByGlobalTID;
	
static javax.transaction.xa.Xid[]getInDoubtXids(javax.transaction.xa.XAResource xaResource)
This method returns InDoubt Xids for a given XAResource

	 if(_logger.isLoggable(Level.FINE))
	 {
	     _logger.logp(Level.FINE,"RecoveryManager", "getInDoubtXids()",
	      "Before receiving inDoubtXids from xaresource = " +
	      xaResource);
         }
         Xid[] inDoubtXids = null;
         ArrayList<Xid> inDoubtXidList = null;
         int flags;
         String recoveryScanFlags = System.getProperty("RECOVERSCANFLAGS");
         if (recoveryScanFlags != null && recoveryScanFlags.equals("TMNOFLAGS"))
             flags = XAResource.TMSTARTRSCAN;
         else
             flags = XAResource.TMSTARTRSCAN | XAResource.TMENDRSCAN;
         boolean continueLoop = true;
         while (continueLoop) {
	     try {
                 inDoubtXids = xaResource.recover(flags);
                 if (inDoubtXids == null || inDoubtXids.length == 0)
                     break;
                 if (flags == XAResource.TMSTARTRSCAN || flags ==  XAResource.TMNOFLAGS) {
                      flags = XAResource.TMNOFLAGS;
                      if (inDoubtXidList == null) {
                          inDoubtXidList = new ArrayList<Xid>();
                      }
                      for (int i = 0; i < inDoubtXids.length; i++)
                          inDoubtXidList.add(inDoubtXids[i]);
                 }
                 else {
                     break;
                 }
             } catch (XAException e) {
	        _logger.log(Level.WARNING,"jts.xaexception_in_recovery",e);
                break;
             }

         }
         if (inDoubtXidList != null)
             inDoubtXids = inDoubtXidList.toArray(new Xid[]{});
	 if(_logger.isLoggable(Level.FINE) && (inDoubtXids != null))
	 {
	     String xidList = LogFormatter.convertXidArrayToString(inDoubtXids);
	     _logger.logp(Level.FINE,"RecoveryManager",
	     "getInDoubtXid()",
	     "InDoubtXids returned from xaresource = "+
	      xaResource + "are: " +xidList);
	 }
         return inDoubtXids;
     
static CoordinatorImplgetLocalCoordinator(java.lang.Long localTID)
Returns a reference to the Coordinator object that corresponds to the local identifier passed as a parameter.

param
localTID The local identifier for the transaction.
return
The Coordinator object.
see


        CoordinatorImpl result = (CoordinatorImpl)
            coordsByLocalTID.get(localTID);

        return result;
    
public static byte[]getRestart()
Gets the restart data for the process.

param
return
The restart data.
see


        byte[] result = null;
        LogFile logFile = Configuration.getLogFile();
        if (logFile != null)
          result = logFile.readRestart();

        return result;
    
static java.lang.ThreadgetThreadFromTxMap(GlobalTID tid)
Get the value (thread) for the specified transaction id from the transaction ID map.

return
the value for the transaction id key from the transaction ID map.

           
        return (Thread) transactionIds.get(tid);
    
static com.sun.enterprise.autotxrecovery.TransactionRecoverygetTransactionRecovery()
return the autoTxRecoveryObject

        return txRecovery;
    
private static java.util.EnumerationgetUniqueRMSet(java.util.Enumeration xaResourceList)
Reduce the set of XAResource objects into a unique set such that there is at most one XAResource object per RM.

 
        Vector uniqueRMList = new Vector();
        
        while (xaResourceList.hasMoreElements()) {
            XAResource xaRes = (XAResource) xaResourceList.nextElement();
            int size = uniqueRMList.size();
            boolean match = false;
            for (int i = 0; i < size; i++) { // compare and eliminate duplicates
                XAResource uniqueXaRes = (XAResource) uniqueRMList.elementAt(i);
                try {
                    if (xaRes.isSameRM(uniqueXaRes)) {
                        match = true; 
                        break;
                    }
                } catch (XAException xe) {}
            }
            if (!match) { 
                uniqueRMList.add(xaRes); 
            }
        }
        
        return uniqueRMList.elements();
    
static voidinitialise()
Initialises the static state of the RecoveryManager class.

param
return
see

                    
       

        // If already initialised, return immediately.

        if (initialised) {
            return;
        }

        initialised = true;

        // Perform recovery/resync if necessary.

        if (Configuration.isRecoverable()) {
            resyncThread = new ResyncThread();
	    	if(_logger.isLoggable(Level.FINE))
	    	{
				_logger.logp(Level.FINE,"RecoveryManager","initialise()",
						"Before starting ResyncThread ");
	    	}
            resyncThread.start();
        } else {

            // If the process is non-recoverable, but there is a valid server
            // name,then check for a log file and issue a warning message
            // if one exists.  Also ensure that restart required is set to no.

            if (!Configuration.isAppClientContainer())  {
                String serverName = Configuration.getServerName();
                if (serverName != null && Log.checkFileExists(serverName)) {
					_logger.log(Level.INFO,"jts.log_file_transient_server",serverName);

                }
            }

            // Modify the restart requirement in the repository, and
            // post the event semaphore as there will be no Coordinators
            // requiring resync.

            try {
                recoveryInProgress.post(); // BUGFIX (Ram Jeyaraman)
                resyncComplete(false, false);
            } catch (Throwable exc) {}
        }
    
public static java.lang.BooleanisIncompleteTxRecoveryRequired()

	    if (inCompleteTxMap.isEmpty()) {
                return Boolean.FALSE;
	    }
            else {
                return Boolean.TRUE;
	    }
    
private static voidproceedWithXARecovery()
This method is used to recontruct and register the Resource objects corresponding to in-doubt transactions in the RMs. It is assumed that the XAResource list has already been provided to the Recovery Manager. This method can be called by Recovery Thread as well as any other thread driving recovery of XA Resources.


        /*  This method has been newly added - Ram Jeyaraman */

        Enumeration xaResources = RecoveryManager.uniqueRMSet;
        String manualRecovery =
            Configuration.getPropertyValue(Configuration.MANUAL_RECOVERY);

        // if ManualRecovery property is not set, do not attempt XA recovery.
        if (manualRecovery == null  ||
                !(manualRecovery.equalsIgnoreCase("true"/*#Frozen*/))) {
            return;
        }

        if (Thread.currentThread().getName().equals("JTS Resync Thread"/*#Frozen*/)) {

            if (uniqueRMSetReady != null) {
                try {
                    uniqueRMSetReady.waitEvent();
                    txRecovery.start();
                    txRecovery.raiseFence();
                    xaResources = RecoveryManager.uniqueRMSet;
                } catch (InterruptedException exc) {
					_logger.log(Level.SEVERE,"jts.wait_for_resync_complete_interrupted");
					 String msg = LogFormatter.getLocalizedMessage(_logger,
					 							"jts.wait_for_resync_complete_interrupted");
					  throw  new org.omg.CORBA.INTERNAL(msg);
                }
            }
        }

        // sanity check
        if (xaResources == null) {
            return;
        }

        Vector otsResources = new Vector();
        // Map uniqueXids = new Hashtable();
        Set uniqueXids = new HashSet();

        while (xaResources.hasMoreElements()) {

            XAResource xaResource = (XAResource) xaResources.nextElement();

            // Get the list of XIDs which represent in-doubt transactions
            // for the database.

            Xid[] inDoubtXids = getInDoubtXids(xaResource);
            // uniqueXids.clear();
            if (inDoubtXids == null || inDoubtXids.length == 0) {
                    continue; // No in-doubt xids for this resource.
            }

            for (int i = 0; i < inDoubtXids.length; i++) {

                    // check to see if the xid belongs to this server.

                    String branchQualifier =
                        new String(inDoubtXids[i].getBranchQualifier());
                    String serverName = Configuration.getServerName();
                    
                    if (branchQualifier.startsWith(serverName)) {

                        // check if the xid is a duplicate. i.e., Xids
                        // which have same globalId and branchId are
                        // considered duplicates. Note that the
                        // branchId format is (serverId, rmId). This is
                        // to make sure that at most one OTSResource object
                        // is registered with the coordinator per transaction
                        // per RM.
                        
                        String xidStr = stringifyXid(inDoubtXids[i]);
                        if (!uniqueXids.contains(inDoubtXids[i])) { // unique xid
                            if(_logger.isLoggable(Level.FINE))
                            {
                                _logger.logp(Level.FINE,"RecoveryManager",
                                        "proceedWithXARecovery",
                                        " This xid is UNIQUE " +
                                        inDoubtXids[i]);
                            }

                            uniqueXids.add(inDoubtXids[i]);// add to uniqueList

                            // Create an OTSResource for the in-doubt
                            // transaction and add it to the list. Each
                            // OTSResource represents a RM per transaction.
                            otsResources.addElement(
                                new OTSResourceImpl(inDoubtXids[i],
                                                    xaResource, null
                                                   ).getCORBAObjReference());
                        } else {
                            if(_logger.isLoggable(Level.FINE))
                            {
                                _logger.logp(Level.FINE,"RecoveryManager",
                                            "proceedWithXARecovery",
                                            " This xid is NOTUNIQUE " +
                                            inDoubtXids[i]);
                            }

                        }
                    } else {
                        if(_logger.isLoggable(Level.FINE))
                        {
                                _logger.logp(Level.FINE,"RecoveryManager",
                                        "proceedWithXARecovery",
                                        " This xid doesn't belong to me " +
                                        inDoubtXids[i]);
                            }

                    }
                }
        }


        // For each OTSResource, determine whether the transaction is known,
        // and if so, register it, otherwise roll it back.

        for (int i = 0; i < otsResources.size(); i++) {

            OTSResource otsResource = (OTSResource) otsResources.elementAt(i);
            GlobalTID globalTID = new GlobalTID(otsResource.getGlobalTID());
            TopCoordinator coord =
                (TopCoordinator) coordsByGlobalTID.get(globalTID);

            if (coord == null) {
                // Roll the OTSResource back if the transaction is not
                // recognised. This happens when the RM has recorded its
                // prepare vote, but the JTS has not recorded its prepare vote.
		    	if(_logger.isLoggable(Level.FINE))
		    	{
					_logger.logp(Level.FINE,"RecoveryManager","proceedWithXARecovery()",
							"Could  not recognize OTSResource: "+otsResource + 
							" with tid: " + 
							LogFormatter.convertToString(globalTID.realTID.tid)+
							";Hence rolling this resource back...");
		    	}
		
               boolean infiniteRetry = true;
               int commitRetries = Configuration.getRetries();
               if (commitRetries >= 0)
                   infiniteRetry = false;
               int commitRetriesLeft = commitRetries;
                boolean exceptionisThrown = true;
                while (exceptionisThrown) {
                    try {
                        otsResource.rollback();
                        exceptionisThrown = false;
                    } catch (Throwable exc) {
                        if ((exc instanceof COMM_FAILURE) || (exc instanceof TRANSIENT)) {
                            if (commitRetriesLeft > 0 || infiniteRetry) {
                                // For TRANSIENT or COMM_FAILURE, wait
                                // for a while, then retry the commit.
                                if (!infiniteRetry) {
                                    commitRetriesLeft--;
                                }

                                try {
                                    Thread.sleep(Configuration.COMMIT_RETRY_WAIT);
                                } catch( Throwable e ) {}
                            }
                            else {
					           _logger.log(Level.WARNING,"jts.exception_during_resync",
							            new java.lang.Object[] {exc.toString(),"OTSResource rollback"});
                               exceptionisThrown = false;
                            }
                        }
                        else {
				            _logger.log(Level.WARNING,"jts.exception_during_resync",
						            new java.lang.Object[] {exc.toString(),"OTSResource rollback"});
                            exceptionisThrown = false;
                        }
                    }
                }
            } else {
                // NOTE: Currently unimplemented. The coordinator needs to
                // check if duplicate resources are being registered for the
                // same RM for the same xid. Also the coordinator should
                // not go away, until all its resources have been sent
                // completion notification. The keypointing should not
                // be done *as is* in the removeCoordinator() method.
                // waitForResync semaphore needs to be flagged when the
                // recovery thread goes away.

                // Register the OTSResource with the Coordinator.
                // It will be called for commit or rollback during resync.
			    	if(_logger.isLoggable(Level.FINE))
	    			{
						_logger.logp(Level.FINE,"RecoveryManager",
								"proceedWithXARecovery()",
								"Recognized OTSResource: " + otsResource + 
								" with tid: " +
								LogFormatter.convertToString(globalTID.realTID.tid) +
								";Hence registering this resource with coordinator...");
	    			}
               		 coord.directRegisterResource(otsResource);
            }
        }
	
static booleanreadAndUpdateTxMap(GlobalTID tid)
Read and update the transaction ID map atomically with the current thread, if and only if there is no concurrent activity for the specified transaction id.

param
tid transaction id.
return
true if there is no concurrent activity and the map has been updated.

        synchronized (transactionIds) {
            Thread thread = (Thread) transactionIds.get(tid);
            if (thread != null) { // concurrent activity
                return false;            
            } 
            // register the thread for the transaction id
            transactionIds.put(tid, Thread.currentThread());                
            return true;
        }        
    
static booleanrecover()
Requests that the RecoveryManager proceed with recovery.

The log is read and a list of TopCoordinators is reconstructed that corresponds to those transactions that were in-doubt at the time of the previous failure.

The method returns true if any transactions require resync.

param
return
Indicates that there are Coordinators requiring resync.
see


        boolean result = false;

        // Check the log for transactions.  If there are any outstanding
        // transactions, recover the Coordinator objects and set up the
        // OMGtid to Coordinator map.

        boolean keypointRequired = false;
        Enumeration logRecords = CoordinatorLog.getLogged();

        while (logRecords.hasMoreElements()) {
            keypointRequired = true;
            try {
                new TopCoordinator().
                    reconstruct((CoordinatorLog) logRecords.nextElement());
            } catch(Exception exc) {
				_logger.log(Level.SEVERE,"jts.recovery_in_doubt_exception",exc);
           		_logger.log(Level.SEVERE,"jts.recovery_in_doubt",exc.toString());
				 String msg = LogFormatter.getLocalizedMessage(_logger,
				 							"jts.recovery_in_doubt",
											new java.lang.Object[] {exc.toString()});
				 throw  new org.omg.CORBA.INTERNAL(msg);
            }
        }

        // Perform recovery of XA resources.

        //recoverXA();
		if(_logger.isLoggable(Level.FINE))
		{
			_logger.logp(Level.FINE,"RecoveryManager","recover()",
					"Before invoking proceedWithXARecovery()");
		}
        proceedWithXARecovery();

        // Post the recovery in progress event so that requests
        // waiting for recovery to complete may proceed.

        recoveryInProgress.post();

        // If resync is not needed, then perform after-resync
        // tasks immediately.

        result = coordsByGlobalTID.size() > 0;
        if (!result) {
            try {
                resyncComplete(false,keypointRequired);
            } catch(Throwable exc) {}
        }

        return result;
    
public static voidrecoverIncompleteTx(javax.transaction.xa.XAResource[] xaresArray)

        if ((xaresArray == null) || (xaresArray.length == 0))
            return;
        int size = xaresArray.length;
        Vector v = new Vector();
        for (int i=0; i<size; i++) {
            v.addElement(xaresArray[i]);
        }
        Enumeration resourceList = getUniqueRMSet(v.elements());
        Set uniqueXids = new HashSet();
        Vector otsResources = new Vector();
        while (resourceList.hasMoreElements()) {
            XAResource xaResource = (XAResource) resourceList.nextElement();
            // Get the list of XIDs which represent in-doubt transactions
            // for the database.
            Xid[] inDoubtXids = getInDoubtXids(xaResource);
            // uniqueXids.clear();
            if (inDoubtXids == null || inDoubtXids.length == 0) {
                continue; // No in-doubt xids for this resource.
            }

            for (int i = 0; i < inDoubtXids.length; i++) {

                    // check to see if the xid belongs to this server.

                    String branchQualifier =
                        new String(inDoubtXids[i].getBranchQualifier());
                    String serverName = Configuration.getServerName();

                    if (branchQualifier.startsWith(serverName)) {

                        // check if the xid is a duplicate. i.e., Xids
                        // which have same globalId and branchId are
                        // considered duplicates. Note that the
                        // branchId format is (serverId, rmId). This is
                                 // to make sure that at most one OTSResource object
                        // is registered with the coordinator per transaction
                        // per RM.

                        String xidStr = stringifyXid(inDoubtXids[i]);
                        if (!uniqueXids.contains(inDoubtXids[i])) { // unique xid
                            if(_logger.isLoggable(Level.FINE))
                            {
                                _logger.logp(Level.FINE,"RecoveryManager",
                                        "recoverIncompleteTx",
                                        " This xid is UNIQUE " +
                                        inDoubtXids[i]);
                            }
                            uniqueXids.add(inDoubtXids[i]); // add to uniqueList
                            otsResources.addElement(
                                new OTSResourceImpl(inDoubtXids[i],
                                                    xaResource, null
                                                   ));
                        } else {
                            if(_logger.isLoggable(Level.FINE))
                            {
                                _logger.logp(Level.FINE,"RecoveryManager",
                                            "recoverIncompleteTx",
                                            " This xid is NOTUNIQUE " +
                                            inDoubtXids[i]);
                            }
                        }
                    } else {
                        if(_logger.isLoggable(Level.FINE))
                        {
                            _logger.logp(Level.FINE,"RecoveryManager",
                                        "recoverIncompleteTx",
                                        " This xid doesn't belong to me " +
                                        inDoubtXids[i]);
                        }
                    }
                }
            } // while (true)
            for (int i = 0; i < otsResources.size(); i++) {
                OTSResourceImpl otsResource = (OTSResourceImpl) otsResources.elementAt(i);
                GlobalTID globalTID = new GlobalTID(otsResource.getGlobalTID());
                synchronized (inCompleteTxMap) {
                    Enumeration e = inCompleteTxMap.keys();
                    while (e.hasMoreElements()) {
                        CoordinatorImpl cImpl = (CoordinatorImpl)e.nextElement();
                        GlobalTID gTID = new GlobalTID(cImpl.getGlobalTID());
                        if (gTID.equals(globalTID)) {
                            Boolean commit = (Boolean) inCompleteTxMap.get(cImpl);
                            boolean infiniteRetry = true;
                            int commitRetries = Configuration.getRetries();
                            if (commitRetries >= 0)
                                infiniteRetry = false;
                            int commitRetriesLeft = commitRetries;
                            boolean exceptionisThrown = true;
                            while (exceptionisThrown) {
                                try {
                                    if (commit.booleanValue()) {
                                        otsResource.commit_one_phase();
                                        if(_logger.isLoggable(Level.FINE)) {
                                            _logger.logp(Level.FINE,"RecoveryManager",
                                                         "recoverIncompleteTx",
                                                         " committed  " +
                                                         otsResource);
                                        }

                                    }
                                    else { 
                                        otsResource.rollback();
                                        if(_logger.isLoggable(Level.FINE)) {
                                            _logger.logp(Level.FINE,"RecoveryManager",
                                                         "recoverIncompleteTx",
                                                         " rolled back  " +
                                                         otsResource);
                                        }

                                    }
                                    exceptionisThrown = false;
                                } catch (Throwable exc) {
                                    if ((exc instanceof COMM_FAILURE) || (exc instanceof TRANSIENT)) {
                                        if (commitRetriesLeft > 0 || infiniteRetry) {
                                            // For TRANSIENT or COMM_FAILURE, wait
                                            // for a while, then retry the commit.
                                            if (!infiniteRetry) {
                                                commitRetriesLeft--;
                                            }
                                            try {
                                                Thread.sleep(Configuration.COMMIT_RETRY_WAIT);
                                            } catch( Throwable iex ) {}
                                        }
                                        else {
					                        _logger.log(Level.WARNING,"jts.exception_during_resync",
							                            new java.lang.Object[] {exc.toString(),"OTSResource rollback"});
                                            exceptionisThrown = false;
                                        }
                                    }
                                    else {
				                        _logger.log(Level.WARNING,"jts.exception_during_resync",
						                            new java.lang.Object[] {exc.toString(),"OTSResource rollback"});
                                        exceptionisThrown = false;
                                    }
                                }
                            }
                        }
                    }
                }
            }
    
public static voidrecoverXAResources(java.util.Enumeration xaResources)
Recovers the in doubt transactions from the provided list of XAResource objects. This method is never called by the recovery thread, and its the application threads which wants to pass in the XA resources that call this.

param
xaResources enumerated list of XA Resources to be recovered


        /*  This method has been newly added - Ram Jeyaraman */

        String manualRecovery =
            Configuration.getPropertyValue(Configuration.MANUAL_RECOVERY);

        // if ManualRecovery property is not set, do not attempt XA recovery.
        if (manualRecovery == null  ||
                !(manualRecovery.equalsIgnoreCase("true"/*#Frozen*/))) {
            return;
        }

        synchronized (lockObject) {

            if (uniqueRMSetReady.isPosted() == false) {
                RecoveryManager.uniqueRMSet = getUniqueRMSet(xaResources);
                uniqueRMSetReady.post();
                waitForResync();
                return;
            } else {
                RecoveryManager.waitForResync();
                RecoveryManager.uniqueRMSet = getUniqueRMSet(xaResources);
                // the following call is meant to induce recovery. But
                // currently it will not work as intended, if it is called
                // during regular TP processing. Currently, this call deals
                // only with XA recovery. There needs to be some support
                // from the coordinator to be able to support recovery
                // during TP processing.
                proceedWithXARecovery();
            }
        }
    
public static voidregisterTransactionRecoveryService(com.sun.enterprise.autotxrecovery.TransactionRecovery txRecoveryService)
Register the implementation of Automatice Transaction recovery. This service is started as soon as all the resources are available.

        txRecovery = txRecoveryService;
    
static booleanremoveCoordinator(GlobalTID globalTID, java.lang.Long localTID, boolean aborted)
Removes the Coordinator associations for the given identifiers.

If there was no association the operation returns false.

Any timeout that was established for the Coordinator is cancelled, and any active thread associations for the transaction are removed and the corresponding Control objects destroyed.

param
globalTID The global identifier for the transaction.
param
localTID The local identifier for the transaction.
param
aborted The transaction aborted indicator.
return
Indicates success of the operation.
see


        boolean result = false;

        // Remove the global identifier to Coordinator mapping if possible.

        CoordinatorImpl coord  = null;
        result = (coordsByGlobalTID.remove(globalTID) != null);

        // Remove the InternalTid to Coordinator mapping if possible.

        if (result) {
            coord = (CoordinatorImpl) coordsByLocalTID.remove(localTID);
            result = (coord != null);
        }

        // If that succeeded, forget the CoordinatorLog object, if the
        // transaction is not a subtransaction.  The following may return
        // FALSE if there are no log records available
        // (i.e. non-recoverable OTS).

        if (coord != null) {
            try {
                if (coord.is_top_level_transaction()) {
                    if (Configuration.isDBLoggingEnabled()) 
                        LogDBHelper.getInstance().deleteRecord(localTID.longValue());
                    else
                        CoordinatorLog.removeLog(localTID);
                }
            } catch(SystemException exc) {
                result = false;
            }
        }

        // Clear the timeout for the transaction, if any.
        // Perform the removal under the timer mutex.

        TimeoutManager.setTimeout(localTID, TimeoutManager.CANCEL_TIMEOUT, 0);

        // Modify any thread associations there may be for the transaction, to
        // indicate that the transaction has ended.



        // COMMENT(Ram J) 09/19/2001 This below line is commented out since in
        // the J2EE controlled environment, all threads are associated and
        // dissociated in an orderly fashion, as well as there is no possibility
        // of concurrent threads active in a given transaction.
        //CurrentTransaction.endAll(globalTID, aborted);

        // If the count of resyncing Coordinators is greater than zero,
        // this means we are still in resync.  Decrease the count.

        if (resyncCoords > 0) {

            resyncCoords--;

            // If the number of resyncing Coordinators is now zero,
            // we may allow new work.

            if (resyncCoords == 0) {
                try {
                    resyncComplete(true, true);
                } catch (Throwable exc) {}
            }
        }

        return result;
    
static java.lang.ThreadremoveFromTxMap(GlobalTID tid)
Remove the specified transaction id from the transaction ID map.

return
the value for the transaction id key from the transaction ID map.

           
        return (Thread) transactionIds.remove(tid);
    
static voidresync()
Performs resync processing.

The RecoveryManager gets recovery information from each TopCoordinator (while holding the transaction lock) and proceeds with resync.

Once resync is complete, a keypoint is taken to indicate that the log information is no longer required.

param
return
see


        // If there are any transactions, proceed with resync.  The map of
        // coordinators by global identifier is created during the
        // TopCoordinator reconstruct method when the coordinators are added
        // via addCoordinator. We copy the contents to another map as
        // Coordinators will remove themselves from the map during resync.

        // Now that the Coordinators have been reconstructed, record
        // the number of transactions requiring resync,
        // and make an event trace point. We must clone the Hashtable
        // here so that the Enumeration does not get
        // changed when any subsequent transaction is created (this can happen
        // when the last Coordinator is removed).

        resyncCoords = coordsByGlobalTID.size();
        Enumeration resyncList =
            ((Hashtable) coordsByGlobalTID.clone()).elements();

        boolean isRoot[] = new boolean[1];

        // Go through and resync each transaction.  The transaction lock
        // for each transaction is obtained to avoid deadlocks during recovery.

        while (resyncList.hasMoreElements()) {

            TopCoordinator coord = (TopCoordinator)resyncList.nextElement();

            try {

                // Before performing recovery, lock the coordinator.

                synchronized (coord) {

                    Status state = coord.recover(isRoot);

                    if (state == Status.StatusUnknown) {

                        // If the coordinator can be locked, then perform
                        // recovery on it. If the outcome is not currently
                        // known, we do nothing with the transaction,
                        // as we expect to eventually get an outcome
                        // from the parent. In this case an in-doubt timeout
                        // is established for the
                        // transaction so that it will continue to retry.
                        // For subordinates, the Coordinator will compl-ete the
                        // transaction itself as it will have no
                        // Synchronization objects.

                        TimeoutManager.setTimeout(
                            new Long(coord.getLocalTID()),
                            TimeoutManager.IN_DOUBT_TIMEOUT,
                            60);

                    } else if (state == Status.StatusCommitted) {

                        // For committed or rolled back, proceed with
                        // completion of the transaction, regardless of
                        // whether it is the root or a subordinate.
                        // If the transaction represents a root, it would
                        // normally wait for the CoordinatorTerm object to
                        // call before completing the transaction.  As there is
                        // no CoordinatorTerm in recovery, we must do it here.
					if(_logger.isLoggable(Level.FINE))
                    {
						_logger.logp(Level.FINE,"RecoveryManager","resync()",
								"Before invoking commit on the reconstructed coordinator"+
								"GTID is: "+
								((TopCoordinator)coord).superInfo.globalTID.toString());
						
                   	}


                        try {
                            coord.commit();
                        } catch (Throwable exc) {
							_logger.log(Level.WARNING,"jts.exception_during_resync",
	                                new java.lang.Object[] {exc.toString(),"commit"});
                        }

                        if (isRoot[0]) {
                            try {
                                coord.afterCompletion(state);
                            } catch (Throwable exc) {
								_logger.log(Level.WARNING,"jts.exception_during_resync",
                               		     new java.lang.Object[] {exc.toString(),
										 "after_completion"});
                            }
                        }

                    } else {

                        // By default, roll the transaction back.

                        try {
							if(_logger.isLoggable(Level.FINE))
                    		{
								_logger.logp(Level.FINE,"RecoveryManager","resync()",
										"Before invoking rollback on the"+
										"reconstructed coordinator :"+
										"GTID is : "+
										((TopCoordinator)coord).superInfo.globalTID.toString());
							
                    		}
                            coord.rollback(true);
                        } catch (Throwable exc) {
							_logger.log(Level.WARNING,"jts.resync_failed",
									new java.lang.Object [] {exc.toString(),"rollback"});
                        }

                        if (isRoot[0]) {
                            try {
                                coord.afterCompletion(Status.StatusRolledBack);
                            } catch (Throwable exc) {
								_logger.log(Level.WARNING,"jts.resync_failed",
										new java.lang.Object[]
                                        { exc.toString(), "after_completion"});
                            }
                        }
                    }
                }
            } catch (Throwable exc) {}
        }

        // Note that resyncComplete will be called by the
        // last TopCoordinator to complete resync (in removeCoordinator)
        // so we do not need to do it here.
    
static voidresyncComplete(boolean resynced, boolean keypointRequired)
Called to indicate that resync is complete.

Indicates that all in-doubt Coordinators recovered from the log have obtained global outcomes are corresponding transactions are complete.

The parameters indicate whether there were Coordinators requiring resync, and whether a keypoint is required.

param
resynced Indicates whether any resync was done.
param
keypointRequired Indicates whether the log needs keypointing.
return
exception
LogicErrorException An internal logic error occurred.
see


        // Inform JTSXA that resync is complete, and trace the fact
        // that resync has completed.

        // COMMENT(Ram J) not needed anymore
        //JTSXA.resyncComplete();

        // Perform a keypoint of the log if required.

        if (keypointRequired) {
            CoordinatorLog.keypoint();
        }

        // Post the resync in progress event semaphore.

        if (resyncInProgress != null) {
            resyncInProgress.post();
            resyncInProgress = null;
        }
    
public static voidsetRestart(byte[] bytes)
Sets the restart data for the process.

param
bytes The restart data.
return
see


        LogFile logFile = Configuration.getLogFile();

        if (logFile != null) {
            if (!logFile.writeRestart(bytes)) {
				_logger.log(Level.WARNING,"jts.restart_write_failed");
            }
        }
    
static voidshutdown(boolean immediate)
Informs the RecoveryManager that the transaction service is being shut down. For immediate shutdown, For quiesce,

param
immediate Indicates whether to stop immediately.
return
see



        /**
        if (immediate) {

            // If immediate, stop the resync thread if any.

            if (resyncThread != null) {
                resyncThread.stop();
            }
        } else {
        **/

            // Otherwise ensure that resync has completed.

            if (resyncInProgress != null) {
                try {
                    resyncInProgress.waitEvent();
                    if (resyncThread != null) {
                        resyncThread.join();
                    }
                } catch (InterruptedException exc) {}
            }
        /**
        }
        **/

        // COMMENT(Ram J) not needed anymore.
        //JTSXA.shutdown(immediate);

        // If not immediate shutdown, keypoint and close the log.
        // Only do this if the process is recoverable!

        if (!immediate && Configuration.isRecoverable()) {
            CoordinatorLog.keypoint();
            CoordinatorLog.finalizeAll();
        }

        //$Continue with shutdown/quiesce.
    
private static java.lang.StringstringifyXid(javax.transaction.xa.Xid xid)

param
xid the xid to be stringified.
return
stringified contents of the xid.

        int glen = xid.getGlobalTransactionId().length;
        int blen = xid.getBranchQualifier().length;
        byte[] xidRep = new byte[glen + 1 + blen];

        System.arraycopy(xid.getGlobalTransactionId(), 0, xidRep, 0, glen);
        xidRep[glen] = (byte) ',";
        System.arraycopy(xid.getBranchQualifier(), 0, xidRep, glen + 1, blen);

        return new String(xidRep);
    
static booleanvalidLocalTID(java.lang.Long localTID)
Determines whether the local transaction identifier represents a valid transaction.

param
localTID The local transaction identifier to check.
return
Indicates the local transaction identifier is valid.
see


        boolean result = coordsByLocalTID.containsKey(localTID);

        return result;
    
public static voidwaitForRecovery()
Waits for recovery to complete.

param
return
see


        if (recoveryInProgress != null) {
            try {
                recoveryInProgress.waitEvent();
            } catch (InterruptedException exc) {
				_logger.log(Level.SEVERE,"jts.wait_for_resync_complete_interrupted");
				 String msg = LogFormatter.getLocalizedMessage(_logger,
											"jts.wait_for_resync_complete_interrupted");
				  throw  new org.omg.CORBA.INTERNAL(msg);
            }
        }
    
public static voidwaitForResync()
Waits for resync to complete.

param
return
see


        if (resyncInProgress != null) {
            try {
                resyncInProgress.waitEvent();
            } catch (InterruptedException exc) {
				_logger.log(Level.SEVERE,"jts.wait_for_resync_complete_interrupted");
				 String msg = LogFormatter.getLocalizedMessage(_logger,
											"jts.wait_for_resync_complete_interrupted");
				  throw  new org.omg.CORBA.INTERNAL(msg);
            }
        }