sm.h

/* -*- mode:C++; c-basic-offset:4 -*-
     Shore-MT -- Multi-threaded port of the SHORE storage manager
   
                       Copyright (c) 2007-2009
      Data Intensive Applications and Systems Labaratory (DIAS)
               Ecole Polytechnique Federale de Lausanne
   
                         All Rights Reserved.
   
   Permission to use, copy, modify and distribute this software and
   its documentation is hereby granted, provided that both the
   copyright notice and this permission notice appear in all copies of
   the software, derivative works or modified versions, and any
   portions thereof, and that both notices appear in supporting
   documentation.
   
   This code is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE AUTHORS
   DISCLAIM ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER
   RESULTING FROM THE USE OF THIS SOFTWARE.
*/

/*<std-header orig-src='shore' incl-file-exclusion='SM_H'>

 $Id: sm.h,v 1.314 2010/07/07 20:50:24 nhall Exp $

SHORE -- Scalable Heterogeneous Object REpository

Copyright (c) 1994-99 Computer Sciences Department, University of
                      Wisconsin -- Madison
All Rights Reserved.

Permission to use, copy, modify and distribute this software and its
documentation is hereby granted, provided that both the copyright
notice and this permission notice appear in all copies of the
software, derivative works or modified versions, and any portions
thereof, and that both notices appear in supporting documentation.

THE AUTHORS AND THE COMPUTER SCIENCES DEPARTMENT OF THE UNIVERSITY
OF WISCONSIN - MADISON ALLOW FREE USE OF THIS SOFTWARE IN ITS
"AS IS" CONDITION, AND THEY DISCLAIM ANY LIABILITY OF ANY KIND
FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

This software was developed with support by the Advanced Research
Project Agency, ARPA order number 018 (formerly 8230), monitored by
the U.S. Army Research Laboratory under contract DAAB07-91-C-Q518.
Further funding for this work was provided by DARPA through
Rome Research Laboratory Contract No. F30602-97-2-0247.

*/

#ifndef SM_H
#define SM_H

#include "w_defines.h"

/*  -- do not edit anything above this line --   </std-header>*/

/*
 *  Stuff needed by value-added servers.  NOT meant to be included by
 *  internal SM .c files, except to the extent that they need these
 *  definitions used in the API.
 */

#ifdef __GNUG__
#pragma interface
#endif

#ifndef SM_INT_4_H
#include <sm_int_4.h>
#endif

#ifndef SM_DU_STATS_H
#include <sm_du_stats.h> // declares sm_du_stats_t
#endif

#ifndef SM_STATS_H
#include <smstats.h> // declares sm_stats_info_t and sm_config_info_t
#endif

#ifndef SM_S_H
#include <sm_s.h> // declares key_type_s, rid_t, lsn_t
#endif

#ifndef LEXIFY_H
#include <lexify.h> // declares sortorder with constants
#endif

#ifndef NBOX_H
#include <nbox.h>   // key_info_t contains nbox_t
#endif /* NBOX_H */

#ifndef SORT_S_H
#include <sort_s.h> // declares key_info_t
#endif

/* DOXYGEN Documentation : */

/**\addtogroup LOGSPACE 
 *
 * Updates performed by transactions are logged so that
 * the can be rolled back (in the event of a transaction abort)
 * or restored (in the event of a crash).  Both the old and new values
 * of an updated location are logged.  This allows a steal, no-force
 * buffer management policy, which means the buffer manager is free
 * to write dirty pages to disk at any time and yet does not have
 * to write dirty pages for a a transaction to commit.
 *
 * The log is stored in a set of Unix files, all in the same directory,
 * whose path is determined by a run-time option.
 * The maximum size of the log is also determined by a run-time option.o
 * The proper value of the log size depends on
 * the expected transaction mix.  More specifically, it depends on the
 * age of the oldest (longest running) transaction in the system and
 * the amount of log space used by all active transactions. Here are
 * some general rules to determine the  amount  of  free  log  space
 * available in the system.
 * - Log records between the first log
 *   record generated by the oldest active transaction and the most
 *   recent log record generated by any transaction cannot be thrown
 *   away.
 * - Log records from a transaction are no longer needed
 *   once the transaction has committed or completely aborted and all
 *   updates have made it to disk. Aborting a transaction causes log space
 *   to be used, so space is reserved for aborting each transaction.
 *   Enough log space must be available to commit or abort all active
 *   transactions at all times.
 * 
 * - Only space starting at the beginning of the log can be reused.  
 *   This space can be reused if it contains log records only for 
 *   transactions meeting the previous rule.
 *
 * -  All storage manager calls that update records require log space twice
 *    the size of the space updated in the record. All calls that create,
 *    append, or truncate records require log space equal to the size
 *    created, inserted, or deleted. Log records generated by these calls
 *    (generally one per call) have an overhead of approximately 50 bytes.
 *
 * - The amount of log space reserved for aborting a transaction is equal to 
 *   the amount of log space generated by the transaction plus a fudge 
 *   factor. 
 *   (Where btrees are concerned, a structure modification
 *   might be necessary on abort, using more space on abort, or might not be
 *   necessary on abort where it was done during forward processing, 
 *   using less space on abort.)
 *
 * - The transaction assumes responsiblity for reserving space in the
 *   log so that it can abort, should it need to (without leaving an
 *   unrecoverable volume).  The transaction and the log cooperate to
 *   reserve space for the transaction's aborting.
 *
 * - When insufficient log space is available for a transaction, the 
 *   transaction is (may be, depending on the server) aborted.
 *   The storage manager will return an error indication (out of log space)
 *   if it is unable to insert a log record into the log due to
 *   insufficient space.
 *
 *   \bug GNATS 142 There remain a number of places in the storage manager code
 *   that react to a lack of log space with a fatal error; this is a
 *   hold-over from the original storage manager, before any attempt to
 *   reserve space was in place.  This code has to be rewritten to handle more
 *   gracefully such errors.  In order for this to be done, the
 *   multi-threaded transaction support will be deprecated.
 *
 * Checkpoints are taken periodically by the storage manager in order to 
 * free log space and shorten recovery time.  Checkpoints are "fuzzy" 
 * and can do not require the system to pause while they are completing.
 *
 * See the storage manager constructor ss_m::ss_m for more information
 * about handling out-of-logspace conditions.
 *
 */

/**\addtogroup SSMOPT
 *
 * These are the run-time options for the storage manager.
 *
 * -sm_bufpoolsize : 
 *      - type: number
 *      - description: This is the size of 
 *      the buffer pool in Kb.  Must be large enough to hold at least 32 pages,
 *      so it depends on the configured page size.
 *      - default: none
 *      - required?: yes
 *
 * -sm_hugetlbfs_path
 *      - type: string (full absolute path name)
 *      - description: Needed only if you configured --with-hugetlbfs.
 *      - default: see \ref CONFIGOPT
 *      - required?: no
 *
 * -sm_reformat_log
 *      - type: Boolean
 *      - description: If "yes", your log will be clobbered and the storage
 *      manager will start up with an entirely new log.
 *      - default: no
 *      - required?: no
 *
 * -sm_logdir
 *      - type: string (relative or absolutee path name)
 *      - description: Location of the log files.
 *      - default: none
 *      - required?: yes
 *
 * -sm_logbufsize
 *      - type: number
 *      - description: size of log buffer in KB.
 *      Must be greater than or equal to the larger of
 *      (4 times the page size, 64 Kb)
 *      and less than or equal to
 *      128 times the page_size. This is the size of 
 *      the log buffer in Kb.
 *      - default: 128
 *      - required?: no
 *
 * -sm_logsize
 *      - type: number
 *      - description: greater than or equal to 8256 
 *      This is the maximum size of the log in Kb.  It is a function of
 *      the log buffer size, and  the default is the minimum allowable for
 *      the default sm_logbufsize.
 *      - default: 128
 *      - required?: yes
 *
 * -sm_log_warn
 *      - type: number between 0 and 100 (percentage)
 *      - description: percentage of log that, when consumed by active
 *      transactions, triggers a callback warning of potential inability
 *      to roll back.   Should be less than 50.
 *      - default: 45
 *      - required?: no
 *
 * -sm_errlog
 *      - type: string (relative or absolute path name OR - )
 *      - description: Destination for error messages.  If "-" is given,
 *      the destination is stderr.
 *      - default: \b -
 *      - required?: no
 *
 * -sm_errlog_level
 *      - type: string  (one of none|emerg|fatal|internal|error|warning|info|debug)
 *      - description: filter.  Message of this priority or higher are issued to
 *      the error log; messages with lower priority are not issued.
 *      The priorities are listed from high to low. "none" means no logging
 *      will happen.
 *      - default: error
 *      - required?: no
 *
 * -sm_locktablesize : 
 *      - type: number greater than or equal to 64
 *      - description: size of lock manager's hash table will be a prime
 *      number near and greater than the given number.
 *      - default: 64000 (yields a hash table with 65521 buckets)
 *      - required?: no
 *
 * -sm_lock_escalate_to_page_threshold
 *      - type: number greater than or equal to 0
 *      - description: after acquiring this many record locks on a page, the lock
 *      will be escalated to a page lock. A value of 0 disables escalation to a
 *      page lock.
 *      - default: 5
 *      - required?: no
 *
 * -sm_lock_escalate_to_store_threshold
 *      - type: number greater than or equal to 0
 *      - description: after acquiring this many page locks on in a store, 
 *      the lock will be escalated to a store lock. 
 *      A value of 0 disables escalation to a store lock.
 *      - default: 25
 *      - required?: no
 *      
 * -sm_lock_escalate_to_volume_threshold
 *      - type: number greater than or equal to 0
 *      - description: after acquiring this many store locks on in a volume, 
 *      the lock will be escalated to a volume lock. 
 *      A value of 0 disables escalation to a volume lock.
 *      - default: 0
 *      - required?: no
 *
 * -sm_cc_alg
 *      - type: string (one of file | page | record | none)
 *      - description: default locking granularity for file operations.
 *      This can be overridden on a per-transaction basis with
 *      ss_m::set_xct_lock_level().
 *      - default: record
 *      - required?: no
 *
 * -sm_backgroundflush
 *      - type: Boolean
 *      - description: Enables background-flushing of volumes.
 *      Must be set to "yes" for sm_num_page_writers to have any effect.
 *      - default: yes
 *      - required?: no
 *
 * -sm_num_page_writers
 *      - type: number
 *      - description: greater than or equal to 0; this is the number of
 *      background-flushing threads for each volume. If you have 
 *      lots of threads, 
 *      a huge buffer pool, and few volumes, you should increase this.
 *      If sm_backgroundflush is "no", this value is ignored.
 *      - default: 2
 *      - required?: no
 *
 * -sm_prefetch
 *      - type: Boolean
 *      - description: Enables prefetching for scans.
 *      - default: no
 *      - required?: no
 *
 * -sm_logging
 *      - type: Boolean
 *      - description: Allows you to turn off logging for a run of
 *      the storage manager. This is only for experimentation, to
 *      measure logging overhead in a limited way.
 *      Aborts, rollbacks and restart/recovery 
 *      do not work without logging.   Independent concurrent
 *      transactions using btrees might not work without logging (this is
 *      not well-tested).
 *      Each time you start the server, you had better start with a
 *      clean device or a device that resulted from a clean shutdown
 *      of the prior run.
 *      - default: yes
 *      - required?: no
 *
 * -sm_lock_caching
 *      - type: Boolean
 *      - description: Enables caching of transaction locks in transaction.
 *      Can be turned off for experimentation. If no, the default is not
 *      to cache locks, but any transaction can turn on caching for itself
 *      by calling the ss_m method  set_lock_cache_enable(bool enable).
 *      - default: yes
 *      - required?: no
 *
 * \sa  \ref SSMVAS
 */


/**\addtogroup SSMXCT 
 * All storage manager operations on data must be done within the scope of
 * a transaction (ss_m::begin_xct, ss_m::commit_xct, ss_m::abort_xct,
 * ss_m::chain_xct). 
 *
 * A very few storage manager operations, such as formatting a volume, are
 * called outside the scope of a transaction and the storage manager begins
 * its own transaction to do the work.
 *
 * Operations that fail return an error indication and the storage 
 * manager assumes that the server will thereafter abort the 
 * transaction in which the error occurred, when abort is indicated.
 * Abort is indicated when eUSERABORT or eDEADLOCK is returned and 
 * when the erver chooses to abort rather than to work around the problem 
 * (whatever it might be, such as eRETRY).
 *
 * The storage manager does not enforce the aborting of any erroneous
 * transactions except, possibly, those that are in danger of 
 * running out of log space.
 * (This is done with the destructor of the prologue used on each call
 * to the storage manager, see next paragraph).
 *
 * It is always the server's responsibility to abort.
 * When the storage manager 
 * encounters a eLOGSPACEWARN condition (the log hasn't enough
 * space \e at \e this \e moment to abort the running transaction,
 * assuming a 1:1 ration of rollback-logging overhead to forward-processing
 * logging overhead), it does one of two things:
 * - passes the error code eLOGSPACEWARN up the call stack back to the server
 *   if the storage manager was constructed with no log-space-warning callback
 *   argument (see LOG_WARN_CALLBACK_FUNC, ss_m::ss_m).
 * - tries to abort a transaction before passing an error code back up
 *   the call stack to the server. Choosing a victim transaction to abort
 *   is done by the server in its log-space-warning callback function (passed
 *   in on ss_m::ss_m, q.v.
 *   Only if that callback function returns a non-null victim transaction
 *   and returns eUSERABORT does the storage manager abort that victim
 *   before returning eUSERABORT up the call stack. Any other
 *   error code returned by the callback function is just returned up
 *   the call stack.
 *
 * \section LOCKS Locks 
 *
 * The storage manager automatically acquires the 
 * necessary locks when the data are read or written.
 * The locks thus acquired are normally released at the end of a transaction,
 * thus, by default, transactions are two-phase and well-formed (degree 3).
 *
 * \subsection GRAN Lock Granularity
 * The fine-grained locks are normally used for records in files, but
 * provision is made for using coarser-grained locks.  The transaction
 * has a default lock level associated with it,
 * which governs the granularity of locks acquired by the storage manager
 * on behalf of the transaction.
 * The lock manager provides for lock escalation to coarser locks to
 * reduce the locking costs.  See \ref SSMLOCK and smlevel_0::concurrency_t. 
 *
 * Key-value locking is normally used for B+-Trees. (See \ref MOH1.)
 * R*-Trees normally use coarse-granularity locking.
 * The locking protocol used with an index is determined when the
 * index is created.  A transaction may acquire coarse (index-level)
 * locks with explicit calls to the lock manager, but by default, 
 * the granularity/level/protocol associated with the index is used.
 * See smlevel_0::concurrency_t. 
 *
 * \section DISTXCT Distributed Transactions
 * Storage manager transactions may be used as "threads" (to 
 * overload this term) of distributed transactions.  
 * Coordination of 2-phase commit must be done externally,
 * but the storage manager supports preparing the (local) transaction "thread" 
 * for two-phase commit, and it will log the necessary 
 * data for recovering in-doubt transactions.
 *
 * \section ATTACH Threads and Transactions
 * Transactions are not tied to storage manager threads (smthread_t, not
 * to be confused with a local "thread" of a distributed transaction) in any 
 * way other than that a transaction must be \e attached to a
 * thread while any storage manager work is being done on behalf of 
 * that transaction.   This is how the storage manager knows \e which
 * transaction is to acquire the locks and latches, etc.
 * But a thread can attach and detach from transactions at will, so
 * work may be performed by different threads each time the storage
 * manager is called on behalf of a given transaction; this allows the
 * server to keep a pool of threads to perform work and allows them to
 * perform work on behalf of any active transaction.
 *
 * \warning
 * While there are limited circumstances in which multiple threads can be
 * attached to the same transaction \e concurrently and perform storage 
 * manager operations on behalf of that transaction concurrently,
 * which is a hold-over from the original storage manager, this 
 * functionality will be deprecated soon.  The reason for this being
 * removed is that it is extremely difficult to handle errors internally
 * when multiple threads are attached to a transaction because 
 * partial rollback is impossible in the absence of multiple log streams
 * for a transaction.
 *
 * Under no circumstances may a thread attach to more than one transaction
 * at a time.
 *
 *
 * \section EXOTICA Exotica
 * The storage manager also provides 
 * - partial rollback (ss_m::save_work and ss_m::rollback_work), 
 *   which undoes actions but does not release locks,
 * - transaction chaining (ss_m::chain_xct), which commits, but retains locks
 *   and gives them to a new transaction,
 * - lock release (sm_quark_t, ss_m::unlock), allowing less-than-3-degree
 *   transactions.
 *
 *  To reduce the cost (particularly in logging) of loading databases,
 *  the storage manager provides for unlogged loading of stores.
 *  See \ref SSMSTORE.
 */


/**\addtogroup SSMDEBUG 
 *
 * \section DEBUGLEV Build-time Debugging Options
 * At configure time, you can control which debugger-related options
 * (symbols, inlining, etc) with the debug-level options. See \ref CONFIGOPT.
 * \section SSMTRACE Tracing (--enable-trace)
 * When this build option is used, additional code is included in the build to
 * enable some limited tracing.  These C Preprocessor macros apply:
 * -W_TRACE
 *  --enable-trace defines this.
 * -FUNC
 *  Outputs the function name when the function is entered.
 * -DBG 
 *  Outputs the arguments.
 * -DBGTHRD 
 *  Outputs the arguments.
 *
 *  The tracing is controlled by these environment variables:
 *  -DEBUG_FLAGS: a list of file names to trace, e.g. "smfile.cpp log.cpp"
 *  -DEBUG_FILE: name of destination for the output. If not defined, the output
 *    is sent to cerr/stderr.
 *
 * See \ref CONFIGOPT.
 *  \note This tracing is not thread-safe, as it uses streams output.
 * \section SSMENABLERC Return Code Checking (--enable-checkrc)
 * If a w_rc_t is set but not checked with method is_error(), upon destruction the
 * w_rc_t will print a message to the effect "error not checked".
 * See \ref CONFIGOPT.
 *
 */

/** \file sm_vas.h
 * \details
 * This is the include file that all value-added servers should
 * include to get the Shore Storage Manager API.
 *
 */
/********************************************************************/

class page_p;
class xct_t;
class device_m;
class vec_t;
class log_m;
class lock_m;
class btree_m;
class file_m;
class pool_m;
class dir_m;
class chkpt_m;
class lid_m; 
class sm_stats_cache_t;
class option_group_t;
class option_t;
class prologue_rc_t;
class rtree_m;
class sort_stream_i;

/**\addtogroup SSMSP  
 * A transaction may perform a partial rollback using savepoints.
 * The transaction populates a savepoint by calling ss_m::save_work,
 * then it may roll back to that point with ss_m::rollback_work.
 * Locks acquired between the save_work and rollback_work are \e not
 * released.
 */

/**\brief A point to which a transaction can roll back.
 * \ingroup SSMSP
 *\details
 * A transaction an do partial rollbacks with
 * save_work  and rollback_work, which use this class to determine
 * how far to roll back.
 * It is nothing more than a log sequence number for the work done
 * to the point when save_work is called.
 */
class sm_save_point_t : public lsn_t {
public:
    NORET            sm_save_point_t(): _tid(0,0) {};
    friend ostream& operator<<(ostream& o, const sm_save_point_t& p) {
        return o << p._tid << ':' << (const lsn_t&) p;
    }
    friend istream& operator>>(istream& i, sm_save_point_t& p) {
        char ch;
        return i >> p._tid >> ch >> (lsn_t&) p;
    }
    tid_t            tid() const { return _tid; }
private:
    friend class ss_m;
    tid_t            _tid;
};

/**\addtogroup SSMQK  
 * A quark is a marker in the transaction's list of acquired locks.
 * One may release all short-duration locks acquired since the quark was inserted 
 * into the list via sm_quark_t::open().
 * The lock manager modifies the locks acquired inside a quark
 * so that non-extent locks are no longer than short-duration.
 *
 * This is for experimentation only, and is \e not well-tested or supported.
 *
 * How used:
 * \code
 * sm_quark_t *q = new sm_quark_t;
 * q->open();  // inserts marker in transaction's list.
 * ...
 * q->close(); // frees short-duration locks to the marker.
 * delete q;
 * \endcode
 *
 * Deleting the quark without closing it causes it to be closed.
 * Quarks may \e not be used with multi-threaded transactions.
 *
 * Note that if a transaction has multiple threads attached when
 * a thread opens a quark, there is no way to determine where the
 * quark takes effect, and since it affects the locks acquired by
 * all threads of the transaction, it must be used very carefully
 * where multiply-threaded transactions are concerned.
 */

/**\brief List of locks acquired by a transaction since
 * the quark was "opened".   
 * \ingroup SSMQK
 * \details
 * When a quark is closed (by calling close()), 
 * the release_locks parameter indicates if all short-duration read
 * locks acquired during the quark should be released.
 * \note Quarks are an experimental feature for use 
 * as a building block for a more general nested-transaction facility.
 *
 * \internal See lock_x.h
 */
class sm_quark_t {
public:
    NORET            sm_quark_t() {}
    NORET            ~sm_quark_t();

    rc_t            open();
    rc_t            close(bool release=true);

    tid_t            tid()const { return _tid; }
    operator         bool()const { return (_tid != tid_t::null); }
    friend ostream& operator<<(ostream& o, const sm_quark_t& q);
    friend istream& operator>>(istream& i, sm_quark_t& q);

private:
    friend class ss_m;
    tid_t            _tid;

    // disable
    sm_quark_t(const sm_quark_t&);
    sm_quark_t& operator=(const sm_quark_t&);

};

class sm_store_info_t;
class log_entry;
class coordinator;
class tape_t;
/**\brief \b This \b is \b the \b SHORE \b Storage \b Manager \b API.
 *\details
 * Most of the API for using the storage manager is through this
 * interface class.
 */
class ss_m : public smlevel_top 
{
    friend class pin_i;
    friend class sort_stream_i;
    friend class prologue_rc_t;
    friend class log_entry;
    friend class coordinator;
    friend class tape_t;
public:

    typedef smlevel_0::LOG_WARN_CALLBACK_FUNC LOG_WARN_CALLBACK_FUNC;
    typedef smlevel_0::LOG_ARCHIVED_CALLBACK_FUNC LOG_ARCHIVED_CALLBACK_FUNC;
    typedef smlevel_0::ndx_t ndx_t;
    typedef smlevel_0::concurrency_t concurrency_t;
    typedef smlevel_1::xct_state_t xct_state_t;

    typedef sm_store_property_t store_property_t;

#if COMMENT
    //
    // Below is most of the interface for the SHORE Storage Manager.
    // The rest is located in pin.h, scan.h, and smthread.h
    //

    //
    // TEMPORARY FILES/INDEXES
    //
    // When a file or index is created there is a tmp_flag parameter
    // that when true indicates that the file is temporary.
    // Operations on a temporary file are not logged and the
    // file will be gone the next time the volume is mounted.
    //
    // TODO: IMPLEMENTATION NOTE on Temporary Files/Indexes:
    //        Temp files cannot be trusted after transaction abort.
    //            They should be marked for removal.
    //
    // CODE STRUCTURE:
    //    Almost all ss_m functions begin by creating a prologue object
    //    whose constructor and descructor check for many common errors.
    //    In addition most ss_m::OP() functions now call an ss_m::_OP()
    //    function to do the real work.  The ss_m::OP functions should
    //    not be called by other ss_m functions, instead the corresponding
    //    ss_m::_OP function should be used.
    //

#endif /* COMMENT */

  public:
    /**\brief Add storage manager options to the given options group.
     *\ingroup SSMINIT
     *\details
     * @param[in] grp The caller's option group, to which the
     * storage manager's options will be added for processing soon.
     *
     * Before the ss_m constructor can be called, setup_options
     * \b must be called.  This will install the storage manager's options and
     * initialize any that are not required.
     * Once all required options have been set, an ss_m can be constructed.
     *
     *\note This is not thread-safe.  The application (server) must prevent
     * concurrent calls to setup_options.
     */
    static rc_t setup_options(option_group_t* grp);

    /**\brief  Initialize the storage manager.
     * \ingroup SSMINIT
     * \details
     * @param[in] warn   A callback function. This is called 
     * when/if the log is in danger of becoming "too full".
     * @param[in] get   A callback function. This is called 
     * when the storage manager needs an archived log file to be restored.
     *
     * When an ss_m object is created, the storage manager initializes itself
     * and,
     * if the sthreads package has not already been initialized by virtue
     * of an sthread_t running, the sthreads package is initialized now.
     *
     * The log is read and recovery is performed (\ref MHLPS), 
     * and control returns to
     * the caller, after which time
     * storage manager threads (instances of smthread_t) may be constructed and
     * storage manager may be used.
     *
     * The storage manager is used by invoking its static methods.  
     * You may use them as follows:
     * \code
     * ss_m *UNIQ = new ss_m();
     *
     * W_DO(UNIQ->mount_dev(...))
     *     // or
     * W_DO(ss_m::mount_dev(...))
     * \endcode
     * ).
     *
     * Only one ss_m object may be extant at any time. If you try
     * to create another while the one exists, a fatal error will occur
     * (your program will choke with a message about your mistake).
     *
     * The callback argument given to the storage manager constructor
     * is called when the storage manager determines that it is in danger
     * of running out of log space.  Heuristics are used to guess when
     * this is the case.  
     *
     * If the function \a warn archives and removes log files, the function
     * \a get must be provided to restore those log files when the
     * storage manager needs them.
     *
     * For details and examples, see  \ref smlevel_0::LOG_WARN_CALLBACK_FUNC, 
     *  \ref smlevel_0::LOG_ARCHIVED_CALLBACK_FUNC, and 
     *  \ref LOGSPACE.
     */
    ss_m(LOG_WARN_CALLBACK_FUNC warn=NULL, LOG_ARCHIVED_CALLBACK_FUNC get=NULL);

    /**\brief  Shut down the storage manager.
     * \ingroup SSMINIT
     * \details
     * When the storage manager object is deleted, it shuts down.
     * Thereafter it is not usable until another ss_m object is 
     * constructed.
     */
    ~ss_m();

    /**\brief Cause the storage manager's shutting down do be done cleanly 
     * or to simulate a crash.
     * \ingroup SSMINIT
     * \details
     * @param[in] clean   True means shut down gracefully, false means simulate a crash.
     *
     * When the storage manager's destructor is called
     * the buffer pool is flushed to disk, unless this method is called 
     * with \a clean == \e false.
     *
     * \note If this method is used, it
     * must be called after the storage manager is 
     * constructed if it is to take effect. Each time the storage
     * manager is constructed, the state associated with this is set
     * to \e true, i.e., "shut down properly".
     *
     * \note This method is not thread-safe, only one thread should use this
     * at any time, presumably just before shutting down.
     */
    static void         set_shutdown_flag(bool clean);

    /**\brief Notify storage manager when a log file was archived by a
     * LOG_WARN_CALLBACK_FUNC.
     *
     * The arguments:
     * @param[in] logfile   Character string name of file archived.
     */
    static rc_t         log_file_was_archived(const char * logfile);

private:
    void                _construct_once(LOG_WARN_CALLBACK_FUNC x=NULL,
                                           LOG_ARCHIVED_CALLBACK_FUNC y=NULL);
    void                _destruct_once();


public:
    /**\addtogroup SSMXCT
     *
     * All work performed on behalf of a transaction must occur while that
     * transaction is "attached" to the thread that performs the work.
     * Creating a transaction attaches it to the thread that creates the transaction. 
     * The thread may detach from the transaction and attach to another.
     * Multiple threads may attach to a single transaction and do work in certain circumstances.   See \ref SSMMULTIXCT
     *
     * 
     */
    /**\brief Begin a transaction 
     *\ingroup SSMXCT
     * @param[in] timeout   Optional, controls blocking behavior.
     * \details
     *
     * Start a new transaction and "attach" it to this thread. 
     * No running transaction may be attached to this thread.
     * 
     * Storage manager methods that must block (e.g., to acquire a lock) 
     * will use the timeout given.  
     * The default timeout is the one associated with this thread.
     *
     * \sa timeout_in_ms
     */
    static rc_t           begin_xct(
        timeout_in_ms            timeout = WAIT_SPECIFIED_BY_THREAD);

    /**\brief Begin an instrumented transaction. 
     *\ingroup SSMXCT
     * @param[in] stats   Pointer to an allocated statistics-holding structure.
     * @param[in] timeout   Optional, controls blocking behavior.
     * \details
     * No running transaction may be already attached to this thread.
     * A new transaction is started and attached to the running thread.
     *
     * The transaction will be instrumented.
     * This structure is updated by the storage manager whenever a thread
     * detaches from this transaction.  The activity recorded during
     * the time the thread is attached to the transcation will be stored in
     * the per-transaction statistics.
     * \attention It is the client's 
     * responsibility to delete the statistics-holding structure.
     * 
     * Storage manager methods that must block (e.g., to acquire a lock) 
     * will use the timeout given.  
     * The default timeout is the one associated with this thread.
     *
     * \sa timeout_in_ms
     */
    static rc_t           begin_xct(
        sm_stats_info_t*         stats,  // allocated by caller
        timeout_in_ms            timeout = WAIT_SPECIFIED_BY_THREAD);

    /**\brief Begin a transaction and return the transaction id.
     *\ingroup SSMXCT
     * @param[out] tid      Transaction id of new transaction.
     * @param[in] timeout   Optional, controls blocking behavior.
     * \details
     *
     * No running transaction may be attached to this thread.
     * 
     * Storage manager methods that must block (e.g., to acquire a lock) 
     * will use the timeout given.  
     * The default timeout is the one associated with this thread.
     *
     * \sa timeout_in_ms
     */
    static rc_t           begin_xct(
        tid_t&                   tid,
        timeout_in_ms            timeout = WAIT_SPECIFIED_BY_THREAD);

    /**\addtogroup SSM2PC  
     * The storage manager contains support for externally-coordinated
     * transactions that use
     * two-phase-commit with presumed abort.
     * The server must provide the coordination and the coordinator is
     * assumed to have its own stable storage, and it is assumed to recover
     * from failures in a "short time", the precise meaning of which is given below.
     * A prepared transaction, like an active transaction,
     * consumes log space and holds locks.
     * Even if a prepared transaction does not hold locks needed by 
     * other transactions, it consumes resources in a way that can interfere 
     * with other transactions.
     * If a prepared transaction remains in the system for a long time 
     * while other transactions are running, eventually the storage 
     * manager needs the log space used (reserved) by the prepared transaction.
     * A coordinator must resolve its prepared transactions
     * before the storage manager effectively runs out of 
     * log space for other transactions in the system.
     * The amount of time involved is a function of the size of the log
     * and of the demands of the other transactions in the system.
     *
     * For the purpose of this discussion, the portion of a global 
     * transaction that involves a single Shore Storage Manager transaction is 
     * called a thread of the global transaction.
     *
     * A Shore transaction participates as a thread of a global transaction
     * as follows:
     - Start a storage-manager transaction with ss_m::begin_xct.
     - Acquire a global transaction identifier from the coordinator.
     - Indicate to the storage manager that this transaction is a 
     thread of a global transaction, and associate the global transaction 
     identifier with this thread by calling ss_m::enter_2pc.
     - Associate a coordinator with the transaction for recovery 
     purposes, by calling ss_m::set_coordinator.
     - Prepare the thread of the transaction and get the storage manager's 
     vote with ss_m::prepare_xct.  
     It is an error to commit a global transaction thread without first 
     preparing it.  It is an error to do anything else 
     in a transaction after it is prepared, except to end 
     the transaction or retry the prepare (to get the vote again).
     - Convey the vote to the coordinator, and determine the transaction's 
     fate from the coordinator.
     - End the thread with ss_m::commit_xct or ss_m::abort_xct.
     *
     * The storage manager 
     * logs the minimal information required to effect a vote of the
     * transaction threads that are storage manager transactions,
     * and to recover such in-doubt transactions after restart.
     * Thus, after a crash/restart, the server may query the storage manager
     * about in-doubt (prepared) transactions with ss_m::query_prepared_xct,
     * which tells the caller the number and global transaction IDs associated
     * with prepared transactions.
     * Using this, the server contacts the coordinator and resumes the
     * voting.
     * The server may find the local transaction IDs and use ss_m::tid_to_xct
     * to attach these transactions  and to resolve them.
     * 
     * Commit and abort of read-only transactions are the same,
     * as these transactions have no log entries.  Preparing read-only transactions
     * causes them to commit/abort and the vote returned is vote_readonly.
     * Once this vote is communicated to the coordinator and the coordinator
     * records it on stable storage, there is no need to involve this thread in
     * any further processing.  For this reason,
     * read-only transactions do not appear as prepared transactions at
     * recovery time.
     * 
     */

    /**\brief Make the attached transaction a thread of a distributed transaction.
     *\ingroup SSM2PC
     *
     * @param[in] gtid    Global transaction ID to associate with this transaction.  This will be logged when the transaction is prepared.
     * 
     * \note This can be called at most once for a given transaction.
     * The transaction must be attached to the calling thread.
     * No other threads may be attached to the transaction.
     */
    static rc_t           enter_2pc(const gtid_t &gtid); 
    /**\brief Assign a coordinator handle to this distributed transaction.
     *\ingroup SSM2PC
     * @param[in] h      Handle of the coordinator.  Not interpreted by
     * the storage manager.
     *
     * The storage manager associates this server handle with the transaction 
     * so that when the transaction is prepared, this information is 
     * written to the log. Upon recovery, if this transaction is still in doubt,
     * the value-added server can query the 
     * storage manager for in-doubt transactions, get their server handles,
     * and resolve the transactions.
     * See query_prepared_xct and recover_2pc.
     */
    static rc_t           set_coordinator(const server_handle_t &h); 

    /**\brief Prepare a thread of a distributed transaction.
     *\ingroup SSM2PC
     * @param[in] stats     Pointer to an allocated statistics-holding 
     *                      structure.
     * @param[out] vote     This thread's vote.
     *
     * The storage manager will prepare the attached transaction (a thread
     * of a distributed transaction) for commit.
     * If this transaction has performed no logged updates, the 
     * vote returned will be vote_readonly.
     * If this transaction can commit, the vote returned will be vote_commit.
     * If an error occurs during the prepare, the vote will be vote_abort.
     *
     * If the transaction is being instrumented, the 
     * statistics-holding structure will be returned to the caller, 
     * and the caller is responsible for its deallocation.
     */
    static rc_t           prepare_xct(
                            sm_stats_info_t*&         stats, 
                            vote_t&                   vote); 

    /**\brief Prepare a thread of a distributed transaction.
     *\ingroup SSM2PC
     * @param[out] vote     This thread's vote. See \ref w_base_t::vote_t.
     *
     * The storage manager will prepare the attached transaction (a thread
     * of a distributed transaction) for commit.
     * If this transaction has performed no logged updates, the 
     * vote returned will be vote_readonly.
     * If this transaction can commit, the vote returned will be vote_commit.
     * If an error occurs during the prepare, the vote will be vote_abort.
     */
    static rc_t           prepare_xct(vote_t &vote); 

    /**\brief Force the transaction to vote "read-only" in a two-phase commit. 
     *\ingroup SSM2PC
     * \details
     * This will override the storage manager's determination of 
     * whether this thread of a distributed transaction is read-only, which is
     * based on whether the local transaction thread logged anything. This
     * method may be useful if the local transaction rolled back to 
     * a savepoint.
     * See  \ref w_base_t::vote_t.
     */
    static rc_t           force_vote_readonly(); 

    /**\brief Given a global transaction id, find the local prepared 
     * transaction associated with it. 
     *\ingroup SSM2PC
     * @param[in] gtid     A global transaction ID (an opaque quantity 
     * to the storage manager).
     * @param[in] mayblock Not used.
     * @param[out] local   Return the transaction ID of the prepared 
     * SM transaction.
     * \details
     * Searches the transaction list for a prepared transaction with the given
     * global transaction id. If found, it returns a reference to the 
     * local transaction.  The transaction is attached to the running
     * thread before it is returned.
     */
    static rc_t           recover_2pc(const gtid_t & gtid,
        bool                      mayblock,
        tid_t &                   local
        );

    /**\brief  Return the number of prepared transactions.
     *\ingroup SSM2PC
     * @param[out] numtids   The number of in-doubt transactions.
     * \details
     * Used by a server at start-up, after recovery, to find out if
     * there are any in-doubt transactions.  If so, the server must
     * use the second form of query_prepared_xct to find the global
     * transaction IDs of these in-doubt transactions.
     */
    static rc_t           query_prepared_xct(int &numtids);

    /**\brief  Return the global transaction IDs of in-doubt transactions. 
     *\ingroup SSM2PC
     * @param[in] numtids   The number of global transaction ids in the list.
     * @param[in] l   The caller-provided list into which to write the 
     * global transaction-ids.
     * \details
     * Used by a server at start-up, after recovery, to find out the
     * global transaction IDs of the prepared transactions.  The storage
     * manager fills in the first numtids entries of the pre-allocated list.
     * The server may have first called the first form of query_prepared_xct
     * to find out how many such transactions there are after recovery.
     *
     * \attention Read-only transactions 
     * do not appear as in-doubt transactions. Because they did not
     * generate any log records, they will not be "discovered" by analysis.
     * The server must determine that any thread of a global transaction that
     * does not appear to be in doubt was a read-only thread or
     * it never prepared and thus has been aborted.
     * Read-only transactions that were prepared would have voted read-only,
     * and if the coordinator recorded that vote on stable storage, it
     * should not be concerned with these transaction threads any further.
     * If the coordinator does not have this information recorded, the
     * transaction thread could have been an aborted non-read-only transaction,
     * so the coordinator must, in this case, presume that the thread aborted
     * and thus make the global transaction abort.
     */
    static rc_t           query_prepared_xct(int numtids, gtid_t l[]);


    /**\brief Commit a transaction.
     *\ingroup SSMXCT
     * @param[in] lazy   Optional, controls flushing of log.
     * @param[out] plastlsn   If non-null, this is a pointer to a
     *                    log sequence number into which the storage
     *                    manager writes the that of the last log record
     *                    inserted for this transaction.
     * \details
     *
     * Commit the attached transaction and detach it, destroy it.
     * If \a lazy is true, the log is not synced.  This means that
     * recovery of this transaction might not be possible.
     */
    static rc_t           commit_xct(
                                     bool   lazy = false,
                                     lsn_t* plastlsn=NULL);

    /**\brief Commit an instrumented transaction and get its statistics.
     *\ingroup SSMXCT
     * @param[out] stats   Get a copy of the statistics for this transaction.
     * @param[in] lazy   Optional, controls flushing of log.
     * @param[out] plastlsn   If non-null, this is a pointer to a
     *                    log sequence number into which the storage
     *                    manager writes the that of the last log record
     *                    inserted for this transaction.
     * \details
     *
     * Commit the attached transaction and detach it, destroy it.
     * If \a lazy is true, the log is not synced.  This means that
     * recovery of this transaction might not be possible.
     */
    static rc_t            commit_xct(
                                    sm_stats_info_t*& stats, 
                                    bool              lazy = false,
                                    lsn_t*            plastlsn=NULL);

    /**\brief Commit an instrumented transaction and start a new one.
     *\ingroup SSMXCT
     * @param[out] stats   Get a copy of the statistics for the first transaction.
     * @param[in] lazy   Optional, controls flushing of log.
     * \details
     *
     * Commit the attached transaction and detach it, destroy it.
     * Start a new transaction and attach it to this thread.
     * \note \e The \e new 
     * \e transaction \e inherits \e the \e locks \e of \e the \e old 
     * \e transaction.
     *
     * If \a lazy is true, the log is not synced.  This means that
     * recovery of this transaction might not be possible.
     */
    static rc_t            chain_xct(
        sm_stats_info_t*&         stats,    /* in w/new, out w/old */
        bool                      lazy = false);  

    /**\brief Commit a transaction and start a new one, inheriting locks.
     *\ingroup SSMXCT
     * @param[in] lazy   Optional, controls flushing of log.
     * \details
     *
     * Commit the attached transaction and detach it, destroy it.
     * Start a new transaction and attach it to this thread.
     * \note \e The \e new 
     * \e transaction \e inherits \e the \e locks \e of \e the \e old 
     * \e transaction.
     *
     * If \a lazy is true, the log is not synced.  This means that
     * recovery of the committed transaction might not be possible.
     */
    static rc_t            chain_xct(bool lazy = false);  

    /**\brief Abort an instrumented transaction and get its statistics.
     *\ingroup SSMXCT
     * @param[out] stats   Get a copy of the statistics for this transaction.
     * \details
     *
     * Abort the attached transaction and detach it, destroy it.
     */
    static rc_t            abort_xct(sm_stats_info_t*&  stats);
    /**\brief Abort a transaction.
     *\ingroup SSMXCT
     * \details
     *
     * Abort the attached transaction and detach it, destroy it.
     */
    static rc_t            abort_xct();

    /**\brief Populate a save point.
     *\ingroup SSMSP
     * @param[out] sp   An sm_save_point_t owned by the caller.
     *\details
     * Store in sp the needed information to be able to roll back 
     * to this point. 
     * For use with rollback_work.
     * \note Only one thread may be attached to a transaction when this
     * is called.
     */
    static rc_t            save_work(sm_save_point_t& sp);

    /**\brief Roll back to a savepoint.
     *\ingroup SSMSP
     * @param[in] sp   An sm_save_point_t owned by the caller and
     * populated by save_work.
     *\details
     * Undo everything that was 
     * done from the time save_work was called on this savepoint.
     * \note Locks are not freed.
     *
     * \note Only one thread may be attached to a transaction when this
     * is called.
     */
    static rc_t            rollback_work(const sm_save_point_t& sp);

    /**\brief Return the number of transactions in active state.
     *\ingroup SSMXCT
     * \details
     * While this is thread-safe, the moment a value is returned, it could
     * be out of date.
     * Useful only for debugging.
     */
    static w_base_t::uint4_t     num_active_xcts();

    /**\brief Attach the given transaction to the currently-running smthread_t.
     *\ingroup SSMXCT
     * \details
     * It is assumed that the currently running thread is an smthread_t.
     */
    static void           attach_xct(xct_t *x) { me()->attach_xct(x); }

    /**\addtogroup SSMMULTIXCT 
     * 
     * Certain operations may be performed while more than one
     * thread is attached to a transaction (this functionality is
     * soon to be deprecated).
     * Any number of attached threads may be read-only.
     * The kinds of updates that can be made by multiple threads are limited by
     * the need to avoid latch-mutex and latch-latch deadlocks. 
     *
     * There are several reasons for this.
     * 1) The multiple threads are not protected from each other by locks.
     * 2) Interleaving of top-level actions is not supported with rollback;
     * this means that for the duration of a top-level action, a thread needs
     * access to the log that excludes all other threads in 
     * the same transaction.
     *
     * The internal logging protocol is this:
     * T1: latch page, log update. Logging requires acquiring a mutex
     * on the xct's log buffer.
     * T2: performing any top-level action, acquires the mutex on the
     * xct's log buffer before doing the action (latching the page).
     *
     * Thus, anything involving top-level actions is suspect.  B-trees
     * use top-level actions, as does file-page allocation, and creation/
     * destruction of stores (files, indexes).  Thus, just about
     * any kind of concurrent updates on the same page
     * in the same transaction is problematic, and just about any update
     * can result in latching extent-map or store-map pages.
     * This activity could be disallowed by enforcing a strict 
     * rule that at most  one update operation can be going on 
     * in a transaction at any time, however this is too restrictive.
     *
     * Multiple updating threads can
     * work \b if \b the \b data \b are \b partitioned by volume.
     * So a well-behaved server may use multiple-threaded transactions
     * to do updates as long as the updates are on different \b volumes.
     * It might also allow read-only transaction threads to be
     * concurrent with a single updating thread.
     *
     * Savepoints and partial rollback may \e not be used with 
     * multi-threaded transactions. This is not enforced by the storage
     * manager; it is poor behavior on the part of a server.
     * For example, the behavior of the following is undefined:
     * - thread 1: attach, read,      read,   read, ...
     * - thread 2: attach, save work, update, rollback
     * If the two threads are reading and possibly updating the same 
     * data, the results are timing-dependent and could produce a latch-
     * latch or latch-mutex deadlock.
     *
     * Ongoing research at DIAS is investigating ways to extend the usefulness
     * of parallelism within a transaction (multi-threaded transactions).
     * Current thoughts about this are for servers to coordinate multiple 
     * transactions using two-phase commit or an optimized version
     * of commit and abort for groups of local transactions.
     */

    /**\brief Detach any attached from the currently-running smthread_t.
     *\ingroup SSMXCT
     * \details
     * Sever the connection between the running thread and the transaction.
     * This allow the running thread to attach a different 
     * transaction and to perform work in its behalf.
     */
    static void           detach_xct() { xct_t *x = me()->xct();
                                        if(x) me()->detach_xct(x); }

    /**\brief Get the transaction structure for a given a transaction id.
     *\ingroup SSMXCT
     * @param[in] tid   Transaction ID.
     *\details
     * Return a pointer to the storage manager's transaction structure.
     * Can be used with detach_xct and attach_xct.
     */
    static xct_t*          tid_to_xct(const tid_t& tid);
    /**\brief Get the transaction ID for a given a transaction structure.
     *\ingroup SSMXCT
     * @param[in] x   Pointer to transaction structure.
     *\details
     * Return the transaction ID for the given transaction.
     */
    static tid_t           xct_to_tid(const xct_t* x);

    /**\brief Print transaction information to an output stream.
     *\ingroup SSMAPIDEBUG
     * @param[in] o   Stream to which to write the information.
     * \details
     * This is for debugging only, and is not thread-safe. 
     */
    static rc_t            dump_xcts(ostream &o);

    /**\brief Get the transaction state for a given transaction (structure).
     *\ingroup SSMXCT
     * @param[in] x   Pointer to transaction structure.
     * \details
     * Returns the state of the transaction (active, prepared). It is
     * hard to get the state of an aborted or committed transaction, since
     * their structures no longer exist.
     */
    static xct_state_t     state_xct(const xct_t* x);

    /**\brief Return the amount of log this transaction would consume
     * if it rolled back.
     *\ingroup SSMXCT
     *
     * If a transaction aborts with eOUTOFLOGSPACE this function can
     * be used in conjunction with xct_reserve_log_space to
     * pre-allocate the needed amount of log space before retrying.
     */
    static smlevel_0::fileoff_t        xct_log_space_needed();

    /**\brief Require the specified amount of log space to be
     * available for this transaction before continuing.
     *\ingroup SSMXCT
     *
     * If a transaction risks running out of log space it can
     * pre-request some or all of the needed amount before starting in
     * order to improve its chances of success. Other new transactions
     * will be unable to acquire log space before this request is
     * granted (existing ones will be able to commit, unless they also
     * run out of space, because that tends to free up log space and
     * avoids wasting work).
     */
    static rc_t            xct_reserve_log_space(fileoff_t amt);
    
    /**\brief Get the locking granularity for the attached transaction.
     * \ingroup SSMLOCK
     */
    static concurrency_t   xct_lock_level();
    /**\brief Set the default locking level for the attached transaction.
     * \ingroup SSMLOCK
     * \details
     * @param[in] l  The level to use for the balance of this transaction.
     * Legitimate values are t_cc_record,  t_cc_page,  t_cc_file.
     *
     * \note Only one thread may be attached to the transaction when this
     * is called. If more than one thread is attached, a fatal error
     * will ensue.
     */
    static void            set_xct_lock_level(concurrency_t l);

    /**\brief Collect transaction information in a virtual table.
     * \ingroup SSMVTABLE
     * \details
     * @param[out] v  The virtual table to populate.
     * @param[in] names_too  If true, make the 
     *            first row of the table a list of the attribute names.
     *
     * All attribute values will be strings.
     * The virtual table v can be printed with its output operator
     * operator<< for ostreams.
     *
     * \attention Not atomic. Can yield stale data. 
     */
    static rc_t            xct_collect(vtable_t&v, bool names_too=true);

    /**\brief Collect buffer pool information in a virtual table.
     * \ingroup SSMVTABLE
     * \details
     * @param[out] v  The virtual table to populate.
     * @param[in] names_too  If true, make the 
     *            first row of the table a list of the attribute names.
     *
     * \attention Be wary of using this with a large buffer pool.
     *
     * All attribute values will be strings.
     * The virtual table v can be printed with its output operator
     * operator<< for ostreams.
     *
     * \attention Not atomic. Can yield stale data. 
     */
    static rc_t            bp_collect(vtable_t&v, bool names_too=true);

    /**\brief Collect lock table information in a virtual table.
     * \ingroup SSMVTABLE
     * \details
     * @param[out] v  The virtual table to populate.
     * @param[in] names_too  If true, make the 
     *            first row of the table a list of the attribute names.
     *
     * All attribute values will be strings.
     * The virtual table v can be printed with its output operator
     * operator<< for ostreams.
     *
     * \attention Not atomic. Can yield stale data. 
     * Cannot be used in a multi-threaded-transaction context.
     */
    static rc_t            lock_collect(vtable_t&v, bool names_too=true);

    /**\brief Collect thread information in a virtual table.
     * \ingroup SSMVTABLE
     * \details
     * @param[out] v  The virtual table to populate.
     * @param[in] names_too  If true, make the 
     *            first row of the table a list of the attribute names.
     *
     * All attribute values will be strings.
     * The virtual table v can be printed with its output operator
     * operator<< for ostreams.
     *
     * \attention Not thread-safe. Can yield stale data. 
     */
    static rc_t            thread_collect(vtable_t&v, bool names_too=true);

    /**\brief Take a checkpoint.
     * \ingroup SSMAPIDEBUG
     * \note For debugging only!
     *
     * Force the storage manager to take a checkpoint.
     * Checkpoints are fuzzy : they can be taken while most other
     * storage manager activity is happening, even though they have
     * to be serialized with respect to each other, and with respect to
     * a few other activities.
     *
     * This is thread-safe.
     */
    static rc_t            checkpoint();

    /**\brief Force the buffer pool to flush its pages to disk.
     * \ingroup SSMAPIDEBUG
     * @param[in] invalidate   True means discard pages after flush.
     * \note For debugging only!
     * \attention Do not call force_buffers with anything pinned.
     * You may cause latch-latch deadlocks, as this method has
     * to scan the entire buffer pool and possibly EX-latch pages to prevent
     * others from updating while it forces to disk.
     * Since the page-order is essentially random, we cannot
     * preclude latch-latch deadlocks with other threads.
     */
    static rc_t            force_buffers(bool invalidate = false);

    /**\brief Force the buffer pool to flush the volume header page(s)
     * to disk.
     * \ingroup SSMAPIDEBUG
     * @param[in] vid   ID of the volume of interest
     * \note For debugging only!
     * \attention Do not call force_vol_hdr_buffers with anything pinned.
     * You could cause latch-latch deadlocks, as this method has
     * to scan the entire buffer pool and possibly EX-latch some pages.
     * Since the page-order is essentially random, we cannot
     * preclude latch-latch deadlocks with other threads.
     */
    static rc_t            force_vol_hdr_buffers( const vid_t&   vid);

    /**\brief Force the buffer pool to flush to disk all pages
     * for the given store.
     * \ingroup SSMAPIDEBUG
     * @param[in] stid   Store whose pages are to be flushed.
     * @param[in] invalidate   True means discard the pages after flushing.
     * \note For debugging only!
     * \attention Do not call force_store_buffers with anything pinned.
     * You may cause latch-latch deadlocks, as this method has
     * to scan the entire buffer pool and, if invalide==true,
     * EX-latch pages to prevent others from updating 
     * while it forces to disk.
     * Since the page-order is essentially random, we cannot
     * preclude latch-latch deadlocks with other threads.
     */
    static rc_t            force_store_buffers(const stid_t & stid,
                                               bool invalidate);

    /**\cond skip 
     * Do not document. Very un-thread-safe.
     */
    static rc_t            dump_buffers(ostream &o);
    static rc_t            dump_locks(ostream &o);
    static rc_t            dump_locks(); // defaults to std::cout
    static rc_t            dump_exts(ostream &o, 
        vid_t                    v, 
        extnum_t                 start, 
        extnum_t                 end);

    static rc_t            dump_stores(ostream &o, 
        vid_t                    v, 
        int                      start, 
        int                      end);

    static rc_t            dump_histo(ostream &o, bool locked);

    static rc_t            snapshot_buffers(
        u_int&                 ndirty, 
        u_int&                 nclean, 
        u_int&                 nfree,
        u_int&                 nfixed);
    /**\endcond skip */

    /**\brief Get a copy of the statistics from an attached instrumented transaction.
     * \ingroup SSMXCT
     * \details
     * @param[out] stats Returns a copy of the statistics for this transaction.
     * @param[in] reset  If true, the statistics for this transaction will be zeroed.
     */
    static rc_t            gather_xct_stats(
        sm_stats_info_t&       stats, 
        bool                   reset = false);

    /**\brief Get a copy of the global statistics.
     * \ingroup SSMSTATS
     * \details
     * @param[out] stats A pre-allocated structure.
     */
    static rc_t            gather_stats(
        sm_stats_info_t&       stats
        );

    /**\brief Get a copy of configuration-dependent information.
     * \ingroup OPT
     * \details
     * @param[out] info A pre-allocated structure.
     */
    static rc_t            config_info(sm_config_info_t& info);

    /**\brief Set sleep time before I/O operations.
     * \ingroup SSMVOL
     * \details
     * This method sets a milli_sec delay to occur before 
     * each disk read/write operation.  This is for debugging.
     * It is useful in discovering thread sync bugs.
     * This delay applies to all threads.
    */
    static rc_t            set_disk_delay(u_int milli_sec);

    /**\cond skip */
    // TODO : document crash testing facilities
    /**\brief Simulate a crash
     * \details
     * This method tells the log manager to start generating corrupted
     * log records.  This will make it appear that a crash occurred
     * at that point in the log.  A call to this method should be
     * followed immediately by a dirty shutdown of the ssm.
     */
    static rc_t            start_log_corruption();
    /**\endcond skip */

    // Forces a log flush
    static rc_t             sync_log(bool block=true);
    static rc_t             flush_until(lsn_t& anlsn, bool block=true);

    // Allowing to access info about the important lsns (curr and durable)
    static rc_t            get_curr_lsn(lsn_t& anlsn);
    static rc_t            get_durable_lsn(lsn_t& anlsn);


    /*
       Device and Volume Management
       ----------------------------
       A device is either an operating system file or operating system
       device and is identified by a path name (absolute or relative).
       A device has a quota.  In theory, a device may have 
       multiple volumes on it but
       in the current implementation the maximum number of volumes
       is 1.

       A volume is where data is stored.  A volume is identified
       uniquely and persistently by a long volume ID (lvid_t).
       Volumes can be used whenever the device they are located
       on is mounted by the SM.  Volumes have a quota.  The
       sum of the quotas of all the volumes on a device cannot
       exceed the device quota.

       The basic steps to begin using a new device/volume are:
        format_dev: initialize the device
        mount_dev: allow use of the device and all its volumes
        generate_new_lvid: generate a unique ID for the volume
        create_vol: create a volume on the device
     */

    /*
     * Device management functions
     */
     /**\addtogroup SSMVOL 
      * The storage manager was designed to permit multiple \e volumes
      * on a \e device, with \e volume analogous to a Unix \e parition and
      * a \e device analogous to a disk, and the original SHORE contained
      * symmetric peer servers.  
      * However good that intention, multiple volumes on a device were never
      * implemented, and times have changed, and the storage manager no
      * longer has any notion of remote and local volumes.
      * The notion a volume, separate from a device, remains, but may
      * some day disappear.
      *
      * For the time being, a device contains at most one volume. 
      *
     * A device is either an operating system file or 
     * an operating system device (e.g., raw disk partition) and  
     * is identified by a path name (absolute or relative).
     *
     * A device has a quota.  
     * A device is intended to have multiple volumes on it, but
     * in the current implementation the maximum number of volumes
     * is exactly 1.
     *
     * A volume is where data are stored.  
     * Each volume is a header and a set of pages. All pages are
     * the same size (this is a compile-time constant, the default being
     * 8K and sizes up to 64K permissible).
     *
     * A volume is identified uniquely and persistently by a 
     * long volume ID (lvid_t), which is stored in its header.
     * Volumes can be used whenever the device they are located
     * on is mounted by the SM.  
     * Volumes have a quota.  The
     * sum of the quotas of all the volumes on a device cannot
     * exceed the device quota.
     *
     * A volume contains a variety of data structures. All user
     * data reside in \e stores.  A store is a collection of the
     * pages on the volume, allocated in \e extents of a size that
     * is a compile-time constant. (The storage manager has only
     * been tested with an extent-size of 8 pages. The compile-time constant
     * can be changed, but it also requires changes elsewhere in the code
     * to maintain alignment of persistent structures.
     * See the comments in config/shore.def.) Thus, the minimum size
     * of a store is one extent's worth of pages.
     * Larger extents provide better clustering, but more wasted space if
     * small files and small indexes will be common.
     *
     * Stores are identified by a store number (snum_t).
     *
     * Each volume contains a few stores that are "overhead":
     * 0 -- is reserved for an extent map and a store map
     * 1 -- directory (dir_m)
     * 2 -- root index 
     *
     * Beyond that, for each (user) file created, 2 stores are used, one for
     * small objects, one for large objects, and for each index (btree, rtree) 
     * created 1 store is used.
     *
     * Each volume is laid out thus:
     * - volume header, which identifies the number of extents on
     *   the volume, determined when the volume is formatted.
     *   This is always in page 1 of the volume.
     * - store map: some number of pages describing the stores on the volume,
     *   namely, being the heads of linked-lists of extents that make up
     *   the stores. The number of such pages is determined when the
     *   volume is formatted.  The worst case is assumed, which is one
     *   might fill the volume with one-extent stores.
     * - extent map: some number of pages of bitmaps, one bitmap for each 
     *   extent,  describe which pages in the extents are allocated or free.
     * - data pages: the rest of the volume.
     *
     */

    /**\brief Format a device.
     * \ingroup SSMVOL
     * \details
     * @param[in] device   Operating-system file name of the "device".
     * @param[in] quota_in_KB  Quota in kilobytes.
     * @param[in] force If true, format the device even if it already exists.
     *
     * Since raw devices always "exist", \a force should be given as true 
     * for raw devices.
     *
     * A device may not be formatted if it is already mounted.
     *
     * \note This method should \b not 
     * be called in the context of a transaction.
     */
    static rc_t            format_dev(
        const char*            device,
        smksize_t              quota_in_KB,
        bool                   force);
    
    /**\brief Mount a device.
     * \ingroup SSMVOL
     * \details
     * @param[in] device   Operating-system file name of the "device".
     * @param[out] vol_cnt Number of volumes on the device.
     * @param[out] devid  A local device id assigned by the storage manager.
     * @param[in] local_vid A local handle to the (only) volume on the device,
     * to be used when a volume is mounted.  The default, vid_t::null, 
     * indicates that the storage manager can chose a value for this. 
     *
     * \note It is fine to mount a device more than once, as long as device
     * is always the same (you cannot specify a hard link or soft link to
     * an entity mounted under a different path). 
     * Device mounts are \b not reference-counted, so a single dismount_dev
     * renders the volumes on the device unusable.
     *
     * \note This method should \b not 
     * be called in the context of a transaction.
     */
    static rc_t            mount_dev(
        const char*            device,
        u_int&                 vol_cnt,
        devid_t&               devid,
        vid_t                  local_vid = vid_t::null);

    /**\brief Dismount a device.
     * \ingroup SSMVOL
     * \details
     * @param[in] device   Operating-system file name of the "device".
     *
     * \note It is fine to mount a device more than once, as long as device
     * is always the same (you cannot specify a hard link or soft link to
     * an entity mounted under a different path). 
     * Device mounts are \b not reference-counted, so a single dismount_dev
     * renders the volumes on the device unusable.
     *
     * \note This method should \b not 
     * be called in the context of a transaction.
     */

    static rc_t            dismount_dev(const char* device);

    /**\brief Dismount all mounted devices.
     * \ingroup SSMVOL
     *
     * \note This method should \b not 
     * be called in the context of a transaction.
     */
    static rc_t            dismount_all();

    // list_devices returns an array of char* pointers to the names of
    // all mounted devices.  Note that the use of a char*'s is 
    // a temporary hack until a standard string class is available.
    // the char* pointers are pointing directly into the device
    // mount table.
    // dev_cnt is the length of the list returned.
    // dev_list and devid_list must be deleted with delete [] by the
    // caller if they are not null (0).  They should be null
    // if an error is returned or if there are no devices.
    /**\brief Return a list of all mounted devices.
     * \ingroup SSMVOL
     * \details
     * @param[out] dev_list   Returned list of pointers directly into the mount table.
     * @param[out] devid_list   Returned list of associated device ids.
     * @param[out] dev_cnt   Returned number of entries in the two above lists.
     *
     * The storage manager allocates the arrays returned with new[], and the
     * caller must return these to the heap with delete[] if they are not null.
     * They will be null if an error is returned or if no devices are mounted.
     *
     * The strings to which dev_list[*] point are \b not to be deleted by
     * the caller.
     */
    static rc_t            list_devices(
        const char**&            dev_list, 
        devid_t*&                devid_list, 
        u_int&                   dev_cnt);

    /**\brief Return a list of all volume on a device.
     * \ingroup SSMVOL
     * \details
     * @param[in] device   Operating-system file name of the "device".
     * @param[out] lvid_list   Returned list of pointers directly into the mount table.
     * @param[out] lvid_cnt   Returned length of list lvid_list.
     *
     * The storage manager allocates the array lvid_list 
     * with new[], and the
     * caller must return it to the heap with delete[] if it is not null.
     * It will be null if an error is returned. 
     *
     * \note This method should \b not 
     * be called in the context of a transaction.
     */
    static rc_t            list_volumes(
        const char*            device,
        lvid_t*&               lvid_list,
        u_int&                 lvid_cnt
    );

    // get_device_quota the "quota" (in KB) of the device
    // and the amount of the quota allocated to volumes on the device.
    /**\brief Get the device quota.
     * \ingroup SSMVOL
     * \details
     * @param[in] device   Operating-system file name of the "device".
     * @param[out] quota_KB   Returned quota in kilobytes
     * @param[out] quota_used_KB   Returned portion of quota allocated to volumes
     *
     * The quota_used_KB is the portion of the quota allocated to volumes on the device.
     *
     * \note This method \b may 
     * be called in the context of a transaction.
     *
     * \note This method \b may 
     * be called in the context of a transaction.
     */
    static rc_t            get_device_quota(
        const char*             device, 
        smksize_t&              quota_KB, 
        smksize_t&              quota_used_KB);


    /*
     * Volume management functions
     */

    /**\brief Change the fake disk latency before I/Os on this volume, 
     * for debugging purposes
     * \ingroup SSMVOL
     * \details
     * @param[in] vid  The ID of the volume of interest.
     * @param[in] adelay  Nanoseconds to sleep with ::nanosleep()
     *
     * This is for debugging only.
     * Changing the value of the latency for a volume does not enable the
     * delay.
     */
    static rc_t set_fake_disk_latency(vid_t vid, const int adelay);

    /**\brief Enable the fake disk latency before I/Os on this volume, for debugging purposes
     * \ingroup SSMVOL
     * \details
     * @param[in] vid  The ID of the volume of interest.
     *
     * This is for debugging only.
     * When this is enabled, is uses whatever disk latency was set with
     * ss_m::create_vol() or the last applied ss_m::set_fake_disk_latency().
     */
    static rc_t enable_fake_disk_latency(vid_t vid);
    /**\brief Disable the fake disk latency before I/Os on this volume, for debugging purposes
     * \ingroup SSMVOL
     * \details
     * @param[in] vid  The ID of the volume of interest.
     *
     * This is for debugging only.
     */
    static rc_t disable_fake_disk_latency(vid_t vid);


    /**\brief Add a volume to a device.
     * \ingroup SSMVOL
     * \details
     * @param[in] lvid  Long volume id to be used on ss_m::create_vol().
     *
     * This generates a unique volume identifier to be written persistently
     * on the volume when it is formatted.
     * This enables us to avoid the mistake of doubly-mounting a volume.
     * The identifer is constructed from the machine network address and the
     * time of day.
     */
    static rc_t generate_new_lvid(lvid_t& lvid);
     
    /**\brief Add a volume to a device.
     * \ingroup SSMVOL
     * \details
     * @param[in] device_name   Operating-system file name of the "device".
     * @param[in] lvid  Long volume id to use when formatting the new volume.
     * @param[in] quota_KB  Quota in kilobytes.
     * @param[in] skip_raw_init  Do not initialize the volume if on a raw device.
     * @param[in] local_vid Short volume id by which to refer to this volume.
     *            If null, the storage manager will assign one.
     * @param[in] apply_fake_io_latency See ss_m::enable_fake_disk_latency()
     * @param[in] fake_disk_latency See ss_m::set_fake_disk_latency()
     *
     * \note This method should \b not 
     * be called in the context of a transaction.
     *
     * The pages on the volume \b must be zeroed; you can only use
     * \a skip_raw_init = true if you have by some other means
     * already initialized the volume.
     */
    static rc_t            create_vol(
        const char*             device_name,
        const lvid_t&           lvid,
        smksize_t               quota_KB,
        bool                    skip_raw_init = false,
        vid_t                   local_vid = vid_t::null,
        const bool              apply_fake_io_latency = false,
        const int               fake_disk_latency = 0);

    /**\brief Destroy a volume.
     * \ingroup SSMVOL
     * \details
     * @param[in] lvid  Long volume id by which the volume is known.
     *
     * \note This method should \b not 
     * be called in the context of a transaction.
     */
    static rc_t            destroy_vol(const lvid_t& lvid);

    /**\brief Gets the quotas associated with the volume.
     * \ingroup SSMVOL
     * @param[in] lvid  Long volume id by which the volume is known.
     * @param[out] quota_KB  Quota given when the volume was created.
     * @param[out] quota_used_KB  Portion of the quota has been used by
     * allocated extents.
     */
    static rc_t            get_volume_quota(
        const lvid_t&             lvid, 
        smksize_t&                quota_KB, 
        smksize_t&                quota_used_KB);

    /**\cond skip */
    // check_volume_page_types: strictly for debugging/testing
    static rc_t             check_volume_page_types(vid_t vid);
    /**\endcond skip */


    /**\brief Analyze a volume and report statistics regarding disk usage.
     * \ingroup SSMVOL
     * @param[in] vid The volume of interest.
     * @param[out] du The structure that will hold the collected statistics.
     * @param[in] audit If "true", the method acquires a share lock on the
     * volume and then will check assertions about the
     * correctness of the data structures on the volume. 
     * If the audit fails an internal fatal error is generated 
     * to facilitate debugging. (It will generate a core file if your
     * shell permits such.)
     * If "false" an IS lock is acquired, which means that the
     * statistics will be fuzzy.
     *
     * Using the audit feature is useful for debugging.
     * It is the only safe way to use this method.
     * \note The statistics are added to the sm_du_stats_t structure passed in.
     * This structure is not cleared by the storage manager.
     */
    static rc_t            get_du_statistics(
        vid_t                 vid,
        sm_du_stats_t&        du,
        bool                  audit = true); 

    /**\brief Analyze a store and report statistics regarding disk usage.
     * \ingroup SSMVOL
     * @param[in] stid The store of interest.
     * @param[out] du The structure that will hold the collected statistics.
     * @param[in] audit If "true", the method acquires a share lock on the
     * store and then will check assertions about the
     * correctness of the data structures on the store. 
     *
     * Using the audit feature is useful for debugging.
     * It is the only safe way to use this method.
     *
     */
    static rc_t            get_du_statistics(
        const stid_t&        stid, 
        sm_du_stats_t&       du,
        bool                 audit = true);
    

    /**\brief Analyze  a volume and collect brief statistics about its usage.
     * \ingroup SSMVOL
     * @param[in] vid The volume of interest.
     * @param[out] volume_stats The statistics are written here.
     * @param[in] cc Indicates whether the volume is to be locked 
     * by this method. Acceptable values are t_cc_none and t_cc_volume.
     *
     * If no lock is acquired, the method can fail with eRETRY.
     *
     */
    static rc_t            get_volume_meta_stats(
        vid_t                vid,
        SmVolumeMetaStats&   volume_stats,
        concurrency_t        cc = t_cc_none
    );

    /**\brief Analyze  a volume and collect brief statistics about its usage.
     * \ingroup SSMVOL
     * @param[in] vid The volume of interest.
     * @param[in] num_files The size of the array file_stats.
     * @param[out] file_stats Preallocated array of structs into which to
     * write the statistics for the individual files inspected.
     * @param[in] batch_calculate  True means make one pass over the volume.
     * @param[in] cc Indicates whether the volume is to be locked 
     * by this method. Acceptable values are t_cc_none and t_cc_volume.
     *
     * If no lock is acquired and batch_calculate is not set, 
     * the method can fail with eRETRY.
     *
     *
     * If batch_calculate is true then this works by making one pass
     * over the meta data, but it looks at all the meta data.  This
     * should be the faster way to do the analysis when there are 
     * many files, and when files use a large portion of the volume.
     *
     * If batch_calculate is false then each file is updated
     * indidually, only looking at the extent information for that
     * particular file. This requires a pass over the volume for each
     * file. (Seek-wise it is less efficient).
     *
     */
    static rc_t            get_file_meta_stats(
        vid_t                vid,
        w_base_t::uint4_t    num_files,
        SmFileMetaStats*     file_stats,
        bool                 batch_calculate = false,
        concurrency_t        cc = t_cc_none
    );
   
    /**\brief Get the index ID of the root index of the volume.
     * \ingroup SSMVOL
     *
     * @param[in] v Volume of interest.
     * @param[out] iid Store ID of the root index.
     * \details
     *
     * Each volume has a root index, which is a well-known
     * index available to the server for bootstrapping a database.
     *
     */
    static rc_t            vol_root_index(
        const vid_t&        v, 
        stid_t&             iid
    )    { iid.vol = v; iid.store = store_id_root_index; return RCOK; }

    /*****************************************************************
     * storage operations: smfile.cpp
     *****************************************************************/
    /**\addtogroup SSMSTORE 
     * Indexes and files are special cases of "stores".
     * A store is a linked list of extents, and an extent is a
     * contiguous group of pages.  So the store is the structure
     * that holds together an ordered set of pages that can be
     * used by a server and have an identifier (a store ID or stid_t).
     *
     * Indexes and files of records are built on stores.
     *
     * Stores have logging properties and 
     * other metadata associated with them.
     * 
     * The property that determines the logging level of the store is
     * \ref sm_store_property_t.
     *
     * Methods that let you get and change the metatdata are:
     * - ss_m::get_store_property
     * - ss_m::set_store_property
     * - ss_m::get_store_info
     * - \ref snum_t
     *
     * When a transaction deletes a file or index, the deletion of the
     * underlying stores is delayed until the transaction commits so that
     * the pages allocated to the stores remain reserved (lest the
     * transaction aborts). The deleting transaction could, in theory,
     * reuse the pages for another store, but in practice that is not done.
     * Instead, when a store is deleted, the store is marked
     * for deletion an put in a list for the transaction to delete upon
     * commit.   At commit time, stores that have property t_load_file
     * or t_insert_file are converted to t_regular.
     */

    /**\brief Change the store property of a file or index.
     * \ingroup SSMSTORE
     * @param[in] stid   File ID or index ID of the store to change.
     * @param[in] property   Enumeration store_property_t (alias for
     *                   smlevel_3::sm_store_property_t, q.v.)
     *
     * \details
     * The possible uses of store properties are described with 
     * smlevel_3::sm_store_property_t.
     */
    static rc_t            set_store_property(
        stid_t                stid,
        store_property_t      property
        );

    /**\brief Get the store property of a file or index.
     * \ingroup SSMSTORE
     * @param[in] stid   File ID or index ID of the store of interest.
     * @param[in] property   Reference to enumeration store_property_t 
     *                  (alias for smlevel_3::sm_store_property_t, q.v.)
     *
     * \details
     * The possible uses of store properties are described with 
     * smlevel_3::sm_store_property_t.
     */
    static rc_t            get_store_property(
        stid_t                stid,
        store_property_t&     property);

    /**\brief Get various store information of a file or index.
     * \ingroup SSMSTORE
     * @param[in] stid   File ID or index ID of the store of interest.
     * @param[out] info  Reference to sm_store_info_t into which to
     * write the results.
     *
     * \details
     * Get internally stored information about a store.
     */
    static rc_t            get_store_info( 
        const stid_t&         stid, 
        sm_store_info_t&      info);

    //
    // Functions for B+tree Indexes
    //
    /**\addtogroup SSMBTREE 
     * The storage manager supports B+-Tree indexes provide associative access 
     * to data by associating keys with values in 1:1 or many:1 relationships.
     * Keys may be composed of any of the basic C-language types (integer,
     * unsigned, floating-point of several sizes) or
     * variable-length character strings (wide characters are \b not supported).
     *
     * The number of key-value pairs that an index can hold is limited by the
     * space available on the volume containing the index.
     * \anchor max_entry_size 
     * The combined sizes of the key and value must
     * be less than or equal to \ref max_entry_size, which is
     * a function of the page size, and is 
     * such that two entries of this size fit on a page along with all
     * the page and entry metadata.  See sm_config_info_t and ss_m::config_info.
     *
     * The minimum size of a B-Tree index is 8 pages (1 extent).
     *
     * A variety of locking protocols is supported:
     * - none : acquire no locks on the {key,value} pairs in the index,
     *   although an intention lock might be acquired on the index.
     * - kvl : key-value locking See \ref MOH1.  The key or
     *   key-value pair is hashed into a 4-byte value and used with the
     *   given store id to make a lock id.
     * - im : index-management locking See \ref MOH1.  
     *   The "value" portion of
     *   the key-value lock is taken to be a record id, which is used 
     *   for the lock id.
     * - modified kvl : an ad-hoc protocol used by the Paradise project. See \ref MODKVL "the scan_index_i constructor". As with index-management locking, 
     *   the "value" portion of
     *   the key-value lock is taken to be a record id, which is used 
     *   for the lock id.
     * - file : full-index locking.
     *
     * \section key_description Key Types
     * A B+-Tree index key has a type determined when the index is created.
     * All keys are stored in lexicographic format based on an interpretation of
     * the key determined by the key description given when the index is
     * created.
     * Lookups on the B+-Tree then involve a single byte-by-byte
     * comparison of two byte-strings, each composed of its concatenated
     * sub-keys.
     *
     * The key description is a null-terminated string as follows:
     \verbatim
     <key_decription>     ::=  <fixed_len_part>*  <variable_len_part>  |
                               <fixed_len_part>+ 
     <fixed_len_part>     ::=  <type> <len> 
     <variable_len_part>  ::=  <type> '*' <len>
     <type>               ::=  'i' | 'u' | 'f' | 'b' | 'I' | 'U' | 'F' | 'B'
     <len>                ::=   [1-9][0-9]*
     \endverbatim
     * Thus, a key may have any number of fixed-length parts followed by at
     * most one variable-length part.
     *
     * The fixed-length parts (if present) consist of a type and a length.
     *
     * The variable-length part (if present) consists of a type and a length
     * separated by an asterisk, which is what distinguishes a variable-length
     * from a fixed-length part.
     *
     * Types and permissible lengths are:
     * - integer (1,2,4,8)
     * - unsigned (1,2,4,8)
     * - floating (4,8)
     * - uninterpreted byte (any length greater than zero)
     *
     * A capital letter indicates that the key part may be compressed. Only prefix
     * compression is implemented, so it makes sense to compress if the
     * first part of the key is compressible.
     *
     * Examples:
     * - "B40u4u2u2" : 40-byte character string followed by a 4-byte integer,
     *                 a 2-byte integer and a 2-byte integer, such as one might
     *                 use for name.year.mo.day.  The character string is
     *                 prefix-compressed.
     * - "f8"        : an 8-byte floating-point number (double)
     * - "I8B*1000"  : An 8-byte integer followed by an uninterpreted string
     *                 of up to 1000 bytes, all prefix-compressed.
     *
     * \note Wide characters are not supported.
     *
     * This key descriptor is stored in the sm_store_info_t, which is
     * stored on the volume and is available with the method ss_m::get_store_info.
     * Keys are stored in \ref LEXICOFORMAT "lexicographic format". The
     * storage manager knows how to convert all the key types listed above.
     * When duplicates are permitted, the index assumes that the elements
     * are in lexicographic order when searching for a <key,element> pair.
     *
     * \section XXXX1 Bulk Loading 
     * Bulk-loading of all index types is supported. See \ref SSMBULKLD.
     */


    /**\brief Create a B+-Tree index.
     * \ingroup SSMBTREE
     * @param[in] vid   Volume on which to create the index.
     * @param[in] ntype   Type of index. Legitimate values are: 
     *  - t_btree : B+-Tree with duplicate keys allowed
     *  - t_uni_btree : B+-Tree without duplicate keys 
     * @param[in] property Logging level of store. Legitimate values are:
     *  - t_regular
     *  - t_load_file
     *  - t_insert_file
     *  See sm_store_property_t for details.
     * @param[in] key_desc Description of key type.
     *  See \ref key_description for details.
     * @param[in] cc The locking protocol to use with this index. See
     * smlevel_0::concurrency_t and \ref SSMBTREE.
     * @param[out] stid New store ID will be returned here.
     */
    static rc_t            create_index(
                vid_t                 vid, 
                ndx_t                 ntype, 
                store_property_t      property,
                const char*           key_desc,
                concurrency_t         cc, 
                stid_t&               stid
    );

    /**\brief Create a B+-Tree or R*-Tree index.
     * \ingroup SSMBTREE
     *\attention For backward compatibility. Will be deprecated later.
     */
    static rc_t            create_index(
                vid_t                 vid, 
                ndx_t                 ntype, 
                store_property_t      property,
                const char*           key_desc,
                stid_t&               stid
    );

    /**\brief Destroy a B+-Tree index.
     * \ingroup SSMBTREE
     *
     * @param[in] iid  ID of the index to be destroyed.
     */
    static rc_t            destroy_index(const stid_t& iid); 

    /**\brief Bulk-load a B+-Tree index from multiple data sources.
     * \ingroup SSMBULKLD
     *
     * @param[in] stid  ID of the index to be loaded.
     * @param[in] nsrcs  Number of files used for data sources.
     * @param[in] source  Array of IDs of files used for data sources.
     * @param[out] stats  Statistics concerning the load activity will be
     *                     written here.
     * @param[in] sort_duplicates  If "true" the bulk-load will sort
     * duplicates by value.
     * @param[in] lexify_keys  If "true" the keys are assumed not to
     * be in 
     * lexicographic format, and the bulk-load will reformat the key before
     * storing it in the index,
     * otherwise they are assumed already to be in lexicographic format.
     *
     * \anchor LEXICOFORMAT 
     * \b Lexicographic \b format
     * is the translation of numbers 
     * (int, float, double, unsigned, etc) into byte strings
     * such that a lexicographic comparison of the byte strings
     * yields the same result as the numeric comparison of the
     * original data.
     *
     * \note The data must already have been sorted by 
     * key in lexicographic format, but the keys themselves don't have
     * to be in lexicographic format; if the keys are not already in
     * lexicographic format, the \a lexify_keys must be given the value "true".
     *
     * In the case of duplicate keys, the bulk-load will handle the
     * sorting of the elements if \a sort_duplicates is "true"; this
     * sort will be done by a lexicographic comparison of the 
     * byte strings that compose the elements.
     */
    static rc_t            bulkld_index(
        const stid_t&             stid, 
        int                       nsrcs,
        const stid_t*             source,
        sm_du_stats_t&            stats,
        bool                      sort_duplicates = true,
        bool                      lexify_keys = true
    );
    /**\brief Bulk-load a B+-Tree index from a single data source.
     * \ingroup SSMBULKLD
     *
     * @param[in] stid  ID of the index to be loaded.
     * @param[in] source  IDs of file used for data source.
     * @param[out] stats  Statistics concerning the load activity will be
     *                     written here.
     * @param[in] sort_duplicates  If "true" the bulk-load will sort
     * duplicates by value.
     * @param[in] lexify_keys  If "true" the keys are assumed not to
     * be in 
     * lexicographic format, and the bulk-load will reformat the key before
     * storing it in the index,
     * otherwise they are assumed already to be in lexicographic format.
     */
    static rc_t            bulkld_index(
        const stid_t&             stid, 
        const stid_t&             source,
        sm_du_stats_t&            stats,
        bool                      sort_duplicates = true,
        bool                      lexify_keys = true
    );
    /**\brief Bulk-load a B+-Tree index from a single data stream.
     * \ingroup SSMBULKLD
     *
     * @param[in] stid  ID of the index to be loaded.
     * @param[in] sorted_stream  Iterator that serves as the data source.
     * @param[out] stats  Statistics concerning the load activity will be
     *                     written here.
     *
     * See sort_stream_i.
     */
    static rc_t            bulkld_index(
        const stid_t&             stid, 
        sort_stream_i&            sorted_stream,
        sm_du_stats_t&            stats);

    /**\cond skip */
    static rc_t            print_index(stid_t stid);
    /**\endcond skip */

    /**\brief Create an entry in a B+-Tree index.
     * \ingroup SSMBTREE
     *
     * @param[in] stid  ID of the index. 
     * @param[in] key  Key for the association to be created.
     * @param[in] el  Element for the association to be created.
     *
     * The combined sizes of the key and element vectors must
     * be less than or equal to \ref max_entry_size.
     */
    static rc_t            create_assoc(
        stid_t                   stid, 
        const vec_t&             key, 
        const vec_t&             el
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
    );
    /**\brief Remove an entry from a B+-Tree index.
     * \ingroup SSMBTREE
     *
     * @param[in] stid  ID of the index. 
     * @param[in] key   Key of the entry to be removed.
     * @param[in] el   Element (value) of the entry to be removed.
     */
    static rc_t            destroy_assoc(
        stid_t                   stid, 
        const vec_t&             key,
        const vec_t&             el
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
    );
    /**\brief Destroy all entries associated with a key in a B+-Tree index. 
     * \ingroup SSMBTREE
     *
     * @param[in] stid  ID of the index. 
     * @param[in] key   Key of the entries to be removed.
     * @param[out] num_removed   The number of entries removed is returned here.
     */
    static rc_t            destroy_all_assoc(
        stid_t                  stid, 
        const vec_t&            key,
        int&                    num_removed
    );
    /**\brief Find an entry associated with a key in a B+-Tree index. 
     * \ingroup SSMBTREE
     *
     * @param[in] stid  ID of the index. 
     * @param[in] key   Key of the entries to be removed.
     * @param[out] el   Element associated with the given key will be copied into this buffer.
     * @param[in] elen Length of buffer into which the 
     *                  result will be written. If too small, eRECWONTFIT will
     *                  be returned.
     *                 Length of result will be returned here.
     * @param[out] found   True if an entry is found.
     *
     * If the index is not unique (allows duplicates), the first
     * element found with the given key will be returned.
     *
     * To locate all entries associated with a non-unique key, you must
     * use scan_index_i, q.v.. 
     */
    static rc_t            find_assoc(
        stid_t                  stid, 
        const vec_t&            key, 
        void*                   el, 
        smsize_t&               elen, 
        bool&                   found
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
    );

    //
    // Functions for R*tree (multi-dimensional(MD), spatial) Indexes
    //

    /**\addtogroup SSMRTREE 
     *
     * An R-tree is a height-balanced structure designed for indexing
     * multi-dimensional spatial objects.  
     * It stores the minimial bounding box (with 2 or higher dimension) of 
     * a spatial object as the key in the leaf pages.
     * This implementation is a variant of an R-Tree called an R*-Tree, which
     * improves the search performance by using a heuristic for redistributing
     * entries and dynamically reorganizing the tree during insertion.
     *
     * An R*-Tree stores key,value pairs where the key is of type nbox_t
     * and the value is of type vec_t.
     *
     * The number of key-value pairs an index can hold is limited by the space
     * available on the volume containing the index.
     * The minimum size of an R*-tree index is 8 pages.
     *
     * 
     * \note This implementation 
     * uses coarse-grained (index-level) locking and 
     * supports only 2 dimensions and integer coordinates.
     * For information about R*-trees, see the \ref BKSS.
     *
     * Example:
     * \code
     scan_rt_i scan(idx, nbox_t::t_overlap, universe, true);
     bool      eof;
     nbox_t    k;
     char*     e;
     smsize_t  elen;

     for(int i=0; 
             (!(rc = scanp->next(k,e,elen,eof)).is_error() && !eof);
             i++) ;
     cout << "Rtree " << idx << " contains " << i << " entries." << endl;
     \endcode
     * 
     *
     * \section XXXX2 Bulk Loading 
     * Bulk-loading of all index types is supported. See \ref SSMBULKLD.
     */
     /*\example rtree_example.cpp*/


    /**\brief Create an R*-Tree (multi-dimensional spatial) index.
     * \ingroup SSMRTREE
     * @param[in] vid   Volume on which to create the index.
     * @param[in] ntype   Type of index. Legitimate values are: 
     *  - t_rtree : R*-Tree 
     * @param[in] property Logging level of store. Legitimate values are:
     *  - t_temporary
     *  - t_regular
     *  - t_load_file
     *  - t_insert_file
     *  See sm_store_property_t for details.
     * @param[in] dim Number of dimensions of the key.
     * They key type is an nbox_t.
     * See \ref nbox_t for details. 
     * @param[out] stid New store ID will be returned here.
     */
    static rc_t            create_md_index(
        vid_t                   vid, 
        ndx_t                   ntype, 
        store_property_t        property,
        stid_t&                 stid, 
        int2_t                  dim = 2
    );

    /**\brief Destroy an R*-Tree index.
     * \ingroup SSMRTREE
     *
     * @param[in] iid  ID of the index to be destroyed.
     */
    static rc_t            destroy_md_index(const stid_t& iid);

    /**\brief Bulk-load a multi-dimensional index from multiple sources.
     * \ingroup SSMBULKLD
     * @param[in] stid  ID of the index to be loaded.
     * @param[in] nsrcs  Number of files used for data sources.
     * @param[in] source  Array of IDs of files used for data sources.
     * @param[out] stats  Statistics concerning the load activity will be
     *                     written here.
     * @param[in] hff   Heuristic fill factor. Not used.
     * @param[in] hef   Heuristic expansion factor. Not used.
     * @param[in] universe  Universal bounding box of all spatial objects indexed.
    */
    static rc_t            bulkld_md_index(
        const stid_t&             stid, 
        int                       nsrcs,
        const stid_t*             source, 
        sm_du_stats_t&            stats,
        int2_t                    hff=75,
        int2_t                    hef=120,
        nbox_t*                   universe=NULL);

    /**\brief Bulk-load a multi-dimensional index from a single source.
     * \ingroup SSMBULKLD
     * @param[in] stid  ID of the index to be loaded.
     * @param[in] source  ID of file to be used for data source.
     * @param[out] stats  Statistics concerning the load activity will be
     *                     written here.
     * @param[in] hff   Heuristic fill factor. Not used.
     * @param[in] hef   Heuristic expansion factor. Not used.
     * @param[in] universe  Universal bounding box of all spatial objects indexed.
    */
    static rc_t            bulkld_md_index(
        const stid_t&             stid, 
        const stid_t&             source, 
        sm_du_stats_t&            stats,
        int2_t                    hff=75,
        int2_t                    hef=120,
        nbox_t*                   universe=NULL);

    /**\brief Bulk-load a multi-dimensional index from a sorted stream source.
     * \ingroup SSMBULKLD
     * @param[in] stid  ID of the index to be loaded.
     * @param[in] sorted_stream  Input stream that is data source.
     * @param[out] stats  Statistics concerning the load activity will be
     *                     written here.
     * @param[in] hff   Heuristic fill factor. Not used.
     * @param[in] hef   Heuristic expansion factor. Not used.
     * @param[in] universe  Universal bounding box of all spatial objects indexed.
    */
    static rc_t            bulkld_md_index(
        const stid_t&             stid, 
        sort_stream_i&            sorted_stream,
        sm_du_stats_t&            stats,
        int2_t                    hff=75,
        int2_t                    hef=120,
        nbox_t*                   universe=NULL);

    static rc_t            print_md_index(stid_t stid);

    /**\brief Look up an entry in a multi-dimensional index.
     * \ingroup SSMRTREE
     *
     * @param[in] stid  ID of the index. 
     * @param[in] key   Key associated with the entry to look up.
     * @param[out] el   Element associated with the given key will be copied into this buffer.
     * @param[in] elen Length of buffer into which the 
     *                  result will be written. If too small, eRECWONTFIT will
     *                  be returned.
     *                 Length of result will be returned here.
     * @param[out] found   True if an entry is found.
     */
    static rc_t            find_md_assoc(
        stid_t                    stid, 
        const nbox_t&             key, 
        void*                     el, 
        smsize_t&                 elen, 
        bool&                     found);

    /**\brief Create an entry in a multi-dimensional index.
     * \ingroup SSMRTREE
     *
     * @param[in] stid  ID of the index. 
     * @param[in] key  Key for the association to be created.
     * @param[in] el  Element for the association to be created.
    */
    static rc_t            create_md_assoc(
        stid_t                    stid, 
        const nbox_t&             key,
        const vec_t&              el);

    /**\brief Destroy an entry in a multi-dimensional index.
     * \ingroup SSMRTREE
     *
     * @param[in] stid  ID of the index. 
     * @param[in] key   Key of the entry to be removed.
     * @param[in] el   Element (value) of the entry to be removed.
    */
    static rc_t            destroy_md_assoc(
        stid_t                    stid, 
        const nbox_t&             key,
        const vec_t&              el);

    /**\cond skip */
    // for debugging
    static rc_t            draw_rtree(const stid_t& stid, ostream &);
    /**\endcond skip */

    /**\brief Gather usage statistics about an R*-Tree index.
     * \ingroup SSMRTREE
     * @param[in] stid  ID of the index. 
     * @param[out] stat  Usage statistics will be written here.
     * @param[in] size  Number of uint2_t's in the array ovp.
     * @param[out] ovp   Pre-allocated array of integers into which
     * the method will write the overlap percentages for each level of the
     * tree.
     * @param[in] audit If "true", the method 
     * will check assertions about the
     * correctness of the rtree.
     * If the audit fails an internal fatal error is generated 
     * to facilitate debugging. (It will generate a core file if your
     * shell permits such.)
     *
     * \note for debugging
    */
    static rc_t            rtree_stats(
        const stid_t&             stid,
        rtree_stats_t&            stat,
        uint2_t                   size = 0,
        uint2_t*                  ovp = NULL,
        bool                      audit = false);

    /**\addtogroup SSMFILE 
     * You can create, destroy, and scan files of records. You may exert some
     * control over the order in which records appear in the file (a physical
     * scan), but, in general, the storage manager decides where to put records.
     *
     * Pages in a file are slotted pages: Each page contains an array of
     * slots.
     * Records take one of three forms: small, large, and very large.
     * - Small records fit in the slots on the file pages.
     * - Large records are too big to fit on a slotted page, so they are put
     * elsewhere, and the slots point to these records.  Actually, what is
     * in a slot is a small array of page pointers to the data of the large record.
     * - A very large record is one whose slot in the file page contains
     *   a single reference to a page that is an index of data pages.
     *
     * Because records may take these forms, the API for creating records
     * contains the opportunity for you to provide a hint about the ultimate
     * size of the record so that the storage manager can create the proper
     * structure for the record immediately, rather than creating a small
     * record that is soon to be converted to a large, then a very large record
     * by subsequent appends. 
     *
     * All records contain a client-defined header.  This is for the convenience
     * of server-writers.  The header must fit on the slotted page, so it should
     * never be very large.
     *
     * The following methods manipulate files of records and the records found 
     * there.
     *
     * Modules below describe file traversal and
     * appending to files (\ref SSMSCANF), 
     * and pinning individual records in the buffer pool for extended operations 
     * (\ref SSMPIN).
     *
     * \section UNINIT Uninitialized Data
     * The functions create_rec, append_rec, and update_rec can be used to
     * write blocks of data that are all zeroes,  with minimal logging. 
     * This is useful for creating records of known size but with uninitialized data.  
     * The type zvec_t, a special case of vec_t, is for this purpose. 
     * Construct it with only a size, as follows:
     * \code
     * zvec_t zdata(100000);
     * \endcode
     * The underlying logging code recognizes that this is a vector of zeroes and
     * logs only a count, not the data themselves. 
     *
     * \section Errors
     * If an error occurs in the middle of one of these methods that is updating persistent data,
     * the record or file \e could be in an inconsistent state. 
     * The caller has the choice of aborting the transaction or rolling back to the nearest savepoint (see \ref SSMXCT).
     *
     * \sa SSMSCAN, SSMPIN, vec_t, zvec_t, IDs.
     */
    
    /**\brief Create a file of records.
     * \ingroup SSMFILE
     * \details
     * @param[in] vid   Volume on which to create a file.
     * @param[out] fid  Returns (store) ID of the new file here.
     * @param[in] property Give the file the this property.
     * @param[in] cluster_hint Not used. 
     *
     * The cluster hint is included in the API for future use. 
     * It has no effect.
     */
    static rc_t            create_file( 
        vid_t                   vid, 
        stid_t&                 fid,
        store_property_t        property,
        shpid_t                 cluster_hint = 0
    ); 

    /**\brief Destroy a file of records.
     * \ingroup SSMFILE
     * \details
     * @param[in] fid  ID of the file to destroy.
     */
    static rc_t            destroy_file(const stid_t& fid); 

    /**\brief Create a new record.
     * \ingroup SSMFILE
     * \details
     * @param[in] fid  ID of the file in which to create a record.
     * @param[in] hdr  What to put in the record's header.
     * @param[in] len_hint  Hint about how big the record will ultimately be.
     * This is used to determine the initial format of the record. If you plan
     * to append to the record and know that it will ultimately become a large
     * record, it is more efficient to give a size hint that is larger than
     * a page here. Otherwise, the record will be made small (as determined by
     * the size of the parameter \a data ), and subsequent appends will cause 
     * the record to be converted to a large record.
     * @param[in] data  What to put in the record's body. 
     * @param[out] new_rid  ID of the newly created record.
     */
    static rc_t            create_rec(
        const stid_t&            fid, 
        const vec_t&             hdr, 
        smsize_t                 len_hint, 
        const vec_t&             data, 
        rid_t&                   new_rid
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
    ); 

    /**\brief Destroy a record.
     * \ingroup SSMFILE
     * \details
     * @param[in] rid  ID of the record to destroy.
     */
    static rc_t            destroy_rec(const rid_t& rid
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
                                       );

    /**\brief Modify the body of an existing record.
     * \ingroup SSMFILE
     * \details
     * @param[in] rid  ID of the record to modify.
     * @param[in] start  First byte to change.
     * @param[in] data  What to put in the record's body.  
     *
     * This overwrites
     * the existing bytes, starting at the offset \a start through the
     * byte at \a start + \a data.size().
     * This method \b cannot \b be \b used to change the size of a record.
     * Attempting this will result in an error.
     */
    static rc_t            update_rec(
        const rid_t&             rid, 
        smsize_t                 start, 
        const vec_t&             data);

    /**\brief Modify the header of an existing record.
     * \ingroup SSMFILE
     * \details
     * @param[in] rid  ID of the record to modify.
     * @param[in] start  First byte to change.
     * @param[in] hdr  What to put in the record's header.  
     *
     * This overwrites
     * the existing bytes, starting at the offset \a start through the
     * byte at \a start + \a data.size().
     * This method \b cannot \b be \b used to change the size of a record
     * header. There are no methods for appending to or truncating a
     * record header.
     *
     * \sa pin_i::update_rec, \ref SSMPIN
     */
    static rc_t            update_rec_hdr(
        const rid_t&             rid, 
        smsize_t                 start, 
        const vec_t&             hdr);
    // see also pin_i::update_rec*()

    /**\brief Append bytes to a record body.
     * \ingroup SSMFILE
     * \details
     * @param[in] rid  ID of the record to modify.
     * @param[in] data  What to append to the record.
     *
     * \note This appends \b to a record; it does \b not append a record to a file!
     * \sa pin_i::append_rec, \ref SSMPIN
     */
    static rc_t            append_rec(
        const rid_t&             rid, 
        const vec_t&             data
                );

    /**\brief Chop bytes off the end of a record body.
     * \ingroup SSMFILE
     * \details
     * @param[in] rid  ID of the record to modify.
     * @param[in] amount  How many bytes to lop off.
     *
     * \sa pin_i::truncate_rec, \ref SSMPIN
     */
    static rc_t            truncate_rec(
        const rid_t&             rid, 
        smsize_t                 amount
    );

    /**\brief Chop bytes off the end of a record body.
     * \ingroup SSMFILE
     * \details
     * @param[in] rid  ID of the record to modify.
     * @param[in] amount  How many bytes to lop off.
     * @param[out] should_forward  Returns true if the record started out
     * large but is now small as a result of the truncation.  
     * This enables a value-added server to take action in this event,
     * should it so desire.
     *
     * \sa pin_i::truncate_rec, \ref SSMPIN
     */
    static rc_t            truncate_rec(
        const rid_t&             rid, 
        smsize_t                 amount,
        bool&                    should_forward 
    );

#ifdef OLDSORT_COMPATIBILITY
    typedef ssm_sort::key_info_t key_info_t;

    /* old sort physical version */
    /**\brief Sort a file. Deprecated.
     * \details
     */
    static rc_t            sort_file(
        const stid_t&             fid, 
        vid_t                     vid, 
        stid_t&                   sfid, 
        store_property_t          property,
        const key_info_t&         key_info, 
        int                       run_size,
        bool                      ascending = true,
        bool                      unique = false,
        bool                      destructive = false,
        bool                      use_new_sort = true);

    /**\brief Sort a file. Deprecated.
     * \details
     */
    static rc_t            new_sort_file(
        const stid_t&             fid, 
        vid_t                     vid, 
        stid_t&                   sfid, 
        store_property_t          property,
        const key_info_t&         key_info, 
        int                       run_size,
        bool                      ascending = true,
        bool                      unique = false,
        bool                      destructive = false
        );
#endif /* OLDSORT_COMPATIBILITY */

    typedef ssm_sort::sort_keys_t sort_keys_t;

    /* new sort physical version : see notes below */
    /**\brief Sort a file.
     * \ingroup SSMSORT
     * @param[in] fid File to sort.
     * @param[in] sorted_fid File to which to write the results. 
     * @param[in] nvids Size of array \a vid.
     * @param[in] vid Array of IDs of scratch files created by the caller.
     * @param[in] kl See sort_keys_t.
     * @param[in] min_rec_sz Hint of minimum record size in input file.
     * @param[in] run_size Number of pages in buffer pool to use for a run. 
     * @param[in] temp_space Number of pages to use for scratch space.
     * (This limits the amount of memory used by the sort).
     *
     * \details
     * Before you call sort_file, you must create an output file \a sorted_fid
     * into which sort_file will write the results.
     *
     * The sort uses temporary files when the input file contains more records
     * than can fit in one run (determined by \a run_size). These temporary files
     * may be spread across multiple volumes, which is useful if the
     * volumes reside on different spindles.  The arguments \a nvids
     * and \a vid are for indicating the volumes to use for these scratch
     * files.
     *
     * The caller can provide a clue in \a min_rec_size
     * about the minimum record size of the
     * input file, which can help the sort's efficiency.
     *
     * The \a run_size indicates how many buffer-pool pages to use
     * for each run.
     * Since at all times one page is fixed for output, while the rest are 
     * for reading the input in runs, the real run size is \a run_size-1.
     * 
     */
    static rc_t            sort_file(
        const stid_t&            fid,     // input file
        const stid_t&            sorted_fid, // output file 
        int                      nvids,    // array size for vids
        const vid_t*             vid,     // array of vids for temp
                        // files
                        // created by caller--
                        // can be same as input file
        sort_keys_t&            kl, // kl &
        smsize_t                min_rec_sz, // for estimating space use
        int                     run_size,   // # pages to use for a run
        int                     temp_space // # pages VM to use for scratch 
    );

    /**\brief Return the short volume ID of a volume.
     * \ingroup SSMVOL
     *
     * @param[in] lvid Long (persistent) volume ID found on the volume's
     * header.
     * @param[out] vid Short volume ID of a mounted volume.
     */
    static rc_t            lvid_to_vid(
        const lvid_t&          lvid,
        vid_t&                 vid);

    /**\brief Return the long volume ID of a volume.
     * \ingroup SSMVOL
     *
     * @param[in] vid Short volume ID of a mounted volume.
     * @param[out] lvid Long (persistent) volume ID found on the volume's
     * header.
     */
    static rc_t            vid_to_lvid(
        vid_t                  vid,
        lvid_t&                lvid);

    /*****************************************************************
     * Locking related functions
     *
     * NOTE: there are standard conversions from lpid_t, rid_t, and
     *       stid_t to lockid_t, so wherever a lockid_t parameter is
     *         specified a lpid_t, rid_t, or stid_t can be used.
     *
     *****************************************************************/

#if SLI_HOOKS
    /* enable/disable SLI globally for all threads created after this
       point. Does *NOT* disable SLI for existing threads.
     */
    static void            set_sli_enabled(bool enabled);
    static void            set_elr_enabled(bool enabled);

    static rc_t            set_log_features(char const* features);
    static char const*         get_log_features();
#endif

    /**\brief Acquire a lock.
     * \ingroup SSMLOCK
     * @param[in]  n  Lock id of the entity to lock. There are
     * conversions from record ids, volume ids, store ids, and page ids to
     * lockid_t.
     * @param[in]  m  Desired lock mode.  Values: EX, SH.
     * @param[in]  d  Desired duration.  Values: 
     * - t_very_long : Held across transaction boundaries; 
     *             cannot be released by unlock()
     * - t_long : Released at commit; cannot be released by unlock()
     * - t_medium : May be released early by explicit unlock()
     * - t_short  : May be released early by explicit unlock()
     * - t_instant : Not held: acquired and released immediately.  Useful
     *             to see if any other transaction holds an incompatible lock.
     * @param[in]  timeout  Milliseconds willing to block.  See timeout_in_ms.
     *
     * The lock manager is written with these durations in mind, but the
     * only durations used by the storage manager are t_instant and t_long.
     * Medium-duration locks are used internally in a one place.  
     *
     * Durations other than long and instant are not well-tested.
     */
    static rc_t            lock(
        const lockid_t&         n, 
        lock_mode_t             m,
        lock_duration_t         d = t_long,
        timeout_in_ms           timeout = WAIT_SPECIFIED_BY_XCT
    );
    
    /**\brief Release a lock.
     * \ingroup SSMLOCK
     * @param[in]  n  Lock id of the entity to lock. There are
     * conversions from record ids, volume ids, store ids, and page ids to
     * lockid_t.
     */
    static rc_t            unlock(const lockid_t& n);

    /**\brief  Disable lock escalation on the given entity. 
     * \ingroup SSMLOCK
     * @param[in]  n  Lock id of the entity to lock. There are
     * conversions from record ids, volume ids, store ids, and page ids to
     * lockid_t.
     * @param[in]  passOnToDescendants If true, apply this to the descendants
     * of \a n.
     */
    static rc_t            dont_escalate(
        const lockid_t&           n,
        bool                      passOnToDescendants = true
    );

    /**\brief  Find the storage-manager-wide escalation thresholds
     * \ingroup SSMLOCK
     * Default values (used for all transactions until they change
     * their per-transaction thresholds) are determined by the
     * storage-manager-wide options.
     * See \ref SSMOPT.
     */
    static rc_t            get_escalation_thresholds(
        w_base_t::int4_t&        toPage,
        w_base_t::int4_t&        toStore,
        w_base_t::int4_t&        toVolume);

    /**\brief  Change the storage-manager-wide escalation thresholds
     * \ingroup SSMLOCK
     * Default values (used for all transactions until they change
     * their per-transaction thresholds) are determined by the
     * storage-manager-wide options.
     * See \ref SSMOPT.
     */
    static rc_t            set_escalation_thresholds(
        w_base_t::int4_t       toPage,
        w_base_t::int4_t       toStore,
        w_base_t::int4_t       toVolume);

    /**\brief  Find out if the attached transaction has an entity locked.
     * \ingroup SSMLOCK
     * @param[in]  n  Lock id of the entity to lock. There are
     * conversions from record ids, volume ids, store ids, and page ids to
     * lockid_t.
     * @param[out]  m  Mode of lock held. NL if none.
     * @param[in]  implicit If "true" the query will returns a lock mode if
     * an implicit lock is held, otherwise the lock must be held explicitly.
     */
    static rc_t            query_lock(
        const lockid_t&        n, 
        lock_mode_t&           m,
        bool                   implicit = false
    );

    /*****************************************************************
     * Lock Cache related functions
     *
     * Each transaction has a cache of recently acquired locks
     * The following functions control the use of the cache.
     * Note that the functions affect the transaction currently
     * associated with the thread.
     *****************************************************************/
    // turn on(enable=true) or  off/(enable=false) the lock cache 
    // return previous state.
    /**\brief Control  lock caching for attached transaction.
     * \ingroup SSMLOCK
     *
     * @param[in] enable Set to true if you want to turn on lock caching
     * for the attached transaction.  The default is that it is turned on.
     *
     * Only long-duration locks are cached.
     * Lock caching can be turned off by default using the 
     * sm_lock_caching option.  Even with it turned off by default, it
     * can be turned on for a given transcation with this method.
     *
     */
    static rc_t            set_lock_cache_enable(bool enable);

    /**\brief True if lock cache is enabled for the attached transaction 
     * \ingroup SSMLOCK
     *
     * @param[out] enabled Will be set to true if the attached transaction has
     * lock caching enabled, false otherwise.
     */
    static rc_t            lock_cache_enabled(bool& enabled);

private:

    static int _instance_cnt;
    static option_group_t* _options;
    static option_t* _hugetlbfs_path;
    static option_t* _reformat_log;
    static option_t* _prefetch;
    static option_t* _bufpoolsize;
    static option_t* _locktablesize;
    static option_t* _logdir;
    static option_t* _logsize;
    static option_t* _logbufsize;
    static option_t* _error_log;
    static option_t* _error_loglevel;
    static option_t* _lockEscalateToPageThreshold;
    static option_t* _lockEscalateToStoreThreshold;
    static option_t* _lockEscalateToVolumeThreshold;
    static option_t* _cc_alg_option;
    static option_t* _log_warn_percent;
    static option_t* _num_page_writers;
    static option_t* _logging;
    static option_t* _lock_caching_default;


    static rc_t            _set_option_logsize(
        option_t*              opt,
        const char*            value,
        ostream*               err_stream);
    
    static rc_t            _set_option_lock_escalate_to_page(
        option_t*              opt,
        const char*            value,
        ostream*               err_stream);
    
    static rc_t            _set_option_lock_escalate_to_store(
        option_t*              opt,
        const char*            value,
        ostream*               err_stream);
    
    static rc_t            _set_option_lock_escalate_to_volume(
        option_t*              opt,
        const char*            value,
        ostream*               err_stream);
    
    static rc_t            _set_store_property(
        stid_t                stid,
        store_property_t      property);

    static rc_t            _get_store_property(
        stid_t                stid,
        store_property_t&     property);

    static rc_t         _begin_xct(
        sm_stats_info_t*      stats,  // allocated by caller
        tid_t&                tid, 
        timeout_in_ms         timeout);

    static rc_t            _commit_xct(
        sm_stats_info_t*&     stats,
        bool                  lazy,
        lsn_t* plastlsn);

    static rc_t            _prepare_xct(
        sm_stats_info_t*&     stats,
        vote_t&                v);

    static rc_t            _set_coordinator(const server_handle_t &); 
    
    static rc_t            _enter_2pc(const gtid_t &); 
    static rc_t            _force_vote_readonly(); 
    static rc_t            _recover_2pc(const gtid_t &,// in
                                bool    mayblock,
                                tid_t    &    //out -- attached if found(?)
                            );
    static rc_t            _chain_xct(
        sm_stats_info_t*&      stats,
        bool                   lazy);

    static rc_t            _abort_xct(
        sm_stats_info_t*&      stats);

    static rc_t            _save_work(sm_save_point_t& sp);

    static rc_t            _rollback_work(const sm_save_point_t&        sp);
    static rc_t            _mount_dev(
        const char*            device,
        u_int&                 vol_cnt,
        vid_t                  local_vid);

    static rc_t            _dismount_dev(
        const char*            device,
        bool                   dismount_if_locked = true
    );
    static rc_t            _create_vol(
        const char*            device_name,
        const lvid_t&          lvid,
        smksize_t              quota_KB,
        bool                   skip_raw_init,
        const bool             apply_fake_io_latency,
        const int              fake_disk_latency);

    static rc_t            _create_index(
        vid_t                 vid, 
        ndx_t                 ntype, 
        store_property_t      property,
        const char*           key_desc,
        concurrency_t         cc,
        stid_t&               stid
    );

    static rc_t            _destroy_index(const stid_t& iid); 

    static rc_t            _get_store_info( 
        const stid_t  &       stid, 
        sm_store_info_t&      info);

    static rc_t            _bulkld_index(
        const stid_t&         stid,
        int                   nsrcs,
        const stid_t*         source,
        sm_du_stats_t&        stats,
        bool                  sort_duplicates = true,
        bool                  lexify_keys = true
    );

    static rc_t            _bulkld_index(
        const stid_t&          stid, 
        sort_stream_i&         sorted_stream,
        sm_du_stats_t&         stats
    );

    static rc_t            _print_index(const stid_t &iid);

    static rc_t            _create_assoc(
        const stid_t  &        stid, 
        const vec_t&           key, 
        const vec_t&           el
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
    );

    static rc_t            _destroy_assoc(
        const stid_t &        stid, 
        const vec_t&          key,
        const vec_t&          el
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
    );

    static rc_t            _destroy_all_assoc(
        const stid_t&        stid, 
        const vec_t&         key,
        int&                 num_removed
    );
    static rc_t            _find_assoc(
        const stid_t&        stid, 
        const vec_t&         key, 
        void*                el, 
        smsize_t&            elen, 
        bool&                found
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
    );

    // below method overloaded for rtree
    static rc_t            _create_md_index(
        vid_t                 vid, 
        ndx_t                 ntype, 
        store_property_t      property,
        stid_t&               stid, 
        int2_t                dim=2
    );

    static rc_t            _destroy_md_index(const stid_t& iid);

    static rc_t            _destroy_md_assoc(
        stid_t                stid,
        const nbox_t&         key,
        const vec_t&          el);

    static rc_t            _bulkld_md_index(
        const stid_t&         stid, 
        int                   nsrcs,
        const stid_t*         source, 
        sm_du_stats_t&        stats,
        int2_t                hff,           // for rtree only
        int2_t                hef,           // for rtree only
        nbox_t*               universe);// for rtree only

    static rc_t            _bulkld_md_index(
        const stid_t&         stid, 
        sort_stream_i&        sorted_stream,
        sm_du_stats_t&        stats,
        int2_t                hff,           // for rtree only
        int2_t                hef,           // for rtree only
        nbox_t*               universe);// for rtree only

    static rc_t            _print_md_index(stid_t stid);

    static rc_t            _create_md_assoc(
        stid_t                stid, 
        const nbox_t&         key,
        const vec_t&          el);

    static rc_t            _find_md_assoc(
        stid_t                stid, 
        const nbox_t&         key, 
        void*                 el, 
        smsize_t&             elen, 
        bool&                 found);

    //
    // The following functions deal with files of records.
    //
    static rc_t            _destroy_n_swap_file(
        const stid_t&         old_fid,
        const stid_t&         new_fid);

    static rc_t            _create_file(
        vid_t                 vid, 
        stid_t&               fid,
        store_property_t     property,
        shpid_t              cluster_hint = 0
    ); 

    static rc_t            _destroy_file(const stid_t& fid); 

    static rc_t            _create_rec(
        const stid_t&            fid, 
        const vec_t&             hdr, 
        smsize_t                 len_hint, 
        const vec_t&             data, 
        rid_t&                   new_rid
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
        ); 

    static rc_t            _destroy_rec(
        const rid_t&             rid
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
        );

    static rc_t            _update_rec(
        const rid_t&             rid, 
        smsize_t                 start, 
        const vec_t&             data
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
        );

    static rc_t            _update_rec_hdr(
        const rid_t&             rid, 
        smsize_t                 start, 
        const vec_t&             hdr
#ifdef SM_DORA
        , const bool             bIgnoreLocks = false
#endif
        );

    static rc_t            _append_rec(
        const rid_t&             rid, 
        const vec_t&             data
        );

    static rc_t            _truncate_rec(
            const rid_t&         rid, 
            smsize_t             amount,
            bool&                should_forward
        );

    static rc_t            _draw_rtree(const stid_t& stid, ostream &);

    static rc_t            _rtree_stats(
            const stid_t&       stid,
            rtree_stats_t&      stat,
            uint2_t             size,
            uint2_t*            ovp,
            bool                audit
        );

#ifdef OLDSORT_COMPATIBILITY
    /* old sort internal, physical */
    static rc_t            _sort_file(
        const stid_t&           fid, 
        vid_t                   vid, 
        stid_t&                 sfid, 
        store_property_t        property,
        const key_info_t&       key_info, 
        int                     run_size,
        bool                    ascending,
        bool                    unique,
        bool                    destructive
    );
#endif /* OLDSORT_COMPATIBILITY */

    /* new sort internal, physical */
    static rc_t            _sort_file(
        const stid_t&             fid,     // input file
        const stid_t&             sorted_fid, // output file -- 
                        // created by caller--
                        // can be same as input file
        int                      nvids,    // array size for vids
        const vid_t*             vid,     // array of vids for temp
        sort_keys_t&             kl,     // key location info &
        smsize_t                 min_rec_sz, // for estimating space use
        int                      run_size,   // # pages to use for a run
        int                      temp_space //# pages VM to use for scratch 
    );


#ifdef OLDSORT_COMPATIBILITY
    /* internal compatibility old sort-> new sort */
    static rc_t            _new_sort_file(
            const stid_t&         in_fid, 
            const stid_t&         out_fid, 
            const key_info_t&    ki, 
            int                  run_size,
            bool                  ascending, 
            bool                  unique, 
            bool                  keep_orig //!destructive
            ); 
#endif /* OLDSORT_COMPATIBILITY */

    static store_flag_t     _make_store_flag(store_property_t property);
    // reverse function:
    // static store_property_t    _make_store_property(w_base_t::uint4_t flag);
    // is in dir_vol_m

    // this is for df statistics  DU DF
    static rc_t            _get_du_statistics(
        vid_t                  vid, 
        sm_du_stats_t&         du,
        bool                   audit);

    static rc_t            _get_du_statistics(
        const stid_t  &        stid, 
        sm_du_stats_t&         du,
        bool                   audit);

    static rc_t            _get_volume_meta_stats(
        vid_t                  vid,
        SmVolumeMetaStats&     volume_stats,
        concurrency_t          cc);

    static rc_t            _get_file_meta_stats(
        vid_t                  vid,
        w_base_t::uint4_t      num_files,
        SmFileMetaStats*       file_stats,
        bool                   batch_calculate,
        concurrency_t          cc);
};

/**\brief Information about a store that can be queried by the client.
 * \details
 * This information is stored in a store directory on the volume.
 * It can be queried with ss_m::get_store_info.
 */
class sm_store_info_t {
public:
    NORET sm_store_info_t(int len) :
                store(0), stype(ss_m::t_bad_store_t), 
                ntype(ss_m::t_bad_ndx_t), cc(ss_m::t_cc_bad),
                eff(0), large_store(0), root(0),
                nkc(0), keydescrlen(len)
                {  keydescr = new char[len]; }

    NORET ~sm_store_info_t() { if (keydescr) delete[] keydescr; }

    /// store number
    snum_t    store;        
    /// t_index, t_file, ... See ss_m::store_t.
    u_char    stype;        
    /// t_btree, t_rtree,... See ss_m::ndx_t
    u_char    ntype;        
    /// t_cc_kvl, t_cc_record,... See ss_m::concurrency_t
    u_char    cc;         

    /// Unused:
    u_char    eff;        

    /// Store number for associated large-page store, if there is one.
    snum_t    large_store; 
    /// Root page if this is an index.
    shpid_t    root;        
    /// Number of key components if this is an index.
    w_base_t::uint4_t    nkc;  
    /// Size of key description (if this is an index)
    int        keydescrlen;    
    /**\brief Variable length string.
     *
     * He who creates a sm_store_info_t for use with get_store_info()
     * is responsible for allocating enough space for 
     * key descriptors if he expects to find them.
     * See \ref key_description.
     */
    char        *keydescr;    
};


ostream& operator<<(ostream& o, const vid_t& v);
istream& operator>>(istream& i, vid_t& v);
ostream& operator<<(ostream& o, const extid_t& x);
istream& operator>>(istream& o, extid_t &x);
ostream& operator<<(ostream& o, const stid_t& stid);
istream& operator>>(istream& i, stid_t& stid);
ostream& operator<<(ostream& o, const lpid_t& pid);
istream& operator>>(istream& i, lpid_t& pid);
ostream& operator<<(ostream& o, const shrid_t& r);
istream& operator>>(istream& i, shrid_t& r);
ostream& operator<<(ostream& o, const rid_t& rid);
istream& operator>>(istream& i, rid_t& rid);
ostream& operator<<(ostream& o, const sm_stats_info_t& s);
template<class ostream>
ostream& operator<<(ostream& o, const sm_config_info_t& s)
{
    o    << "  page_size " << s.page_size
     << "  max_small_rec " << s.max_small_rec
     << "  lg_rec_page_space " << s.lg_rec_page_space
     << "  buffer_pool_size " << s.buffer_pool_size
     << "  max_btree_entry_size " << s.max_btree_entry_size
     << "  exts_on_page " << s.exts_on_page
     << "  pages_per_ext " << s.pages_per_ext
     << "  multi_threaded_xct " << s.multi_threaded_xct
     << "  logging " << s.logging
      ;
    return o;
}


#ifndef VEC_T_H
#include <vec_t.h>
#endif

#ifndef SM_ESCALATION_H
#include <sm_escalation.h>
#endif

/*<std-footer incl-file-exclusion='SM_H'>  -- do not edit anything below this line -- */

#endif          /*</std-footer>*/

---