00001 /*<std-header orig-src='shore'> 00002 00003 $Id: create_rec.cpp,v 1.4 2010/06/24 17:08:14 nhall Exp $ 00004 00005 SHORE -- Scalable Heterogeneous Object REpository 00006 00007 Copyright (c) 1994-99 Computer Sciences Department, University of 00008 Wisconsin -- Madison 00009 All Rights Reserved. 00010 00011 Permission to use, copy, modify and distribute this software and its 00012 documentation is hereby granted, provided that both the copyright 00013 notice and this permission notice appear in all copies of the 00014 software, derivative works or modified versions, and any portions 00015 thereof, and that both notices appear in supporting documentation. 00016 00017 THE AUTHORS AND THE COMPUTER SCIENCES DEPARTMENT OF THE UNIVERSITY 00018 OF WISCONSIN - MADISON ALLOW FREE USE OF THIS SOFTWARE IN ITS 00019 "AS IS" CONDITION, AND THEY DISCLAIM ANY LIABILITY OF ANY KIND 00020 FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 00021 00022 This software was developed with support by the Advanced Research 00023 Project Agency, ARPA order number 018 (formerly 8230), monitored by 00024 the U.S. Army Research Laboratory under contract DAAB07-91-C-Q518. 00025 Further funding for this work was provided by DARPA through 00026 Rome Research Laboratory Contract No. F30602-97-2-0247. 00027 00028 */ 00029 00030 #include "w_defines.h" 00031 00032 /* -- do not edit anything above this line -- </std-header>*/ 00033 00034 /**\anchor create_rec_example */ 00035 /* 00036 * This program is a simple test of creating records 00037 */ 00038 00039 #include "sm_vas.h" 00040 ss_m* ssm = 0; 00041 00042 // shorten error code type name 00043 typedef w_rc_t rc_t; 00044 00045 // this is implemented in options.cpp 00046 w_rc_t init_config_options(option_group_t& options, 00047 const char* prog_type, 00048 int& argc, char** argv); 00049 00050 00051 struct file_info_t { 00052 static const char* key; 00053 stid_t fid; 00054 rid_t first_rid; 00055 int num_rec; 00056 int rec_size; 00057 }; 00058 const char* file_info_t::key = "SCANFILE"; 00059 00060 ostream & 00061 operator << (ostream &o, const file_info_t &info) 00062 { 00063 o << "key " << info.key 00064 << " fid " << info.fid 00065 << " first_rid " << info.first_rid 00066 << " num_rec " << info.num_rec 00067 << " rec_size " << info.rec_size ; 00068 return o; 00069 } 00070 00071 00072 typedef smlevel_0::smksize_t smksize_t; 00073 00074 00075 00076 void 00077 usage(option_group_t& options) 00078 { 00079 cerr << "Usage: create_rec [-h] [-i] [options]" << endl; 00080 cerr << " -i initialize device/volume and create file of records" << endl; 00081 cerr << "Valid options are: " << endl; 00082 options.print_usage(true, cerr); 00083 } 00084 00085 /* create an smthread based class for all sm-related work */ 00086 class smthread_user_t : public smthread_t { 00087 int _argc; 00088 char **_argv; 00089 00090 const char *_device_name; 00091 smsize_t _quota; 00092 int _num_rec; 00093 smsize_t _rec_size; 00094 lvid_t _lvid; 00095 rid_t _start_rid; 00096 stid_t _fid; 00097 bool _initialize_device; 00098 option_group_t* _options; 00099 vid_t _vid; 00100 public: 00101 int retval; 00102 00103 smthread_user_t(int ac, char **av) 00104 : smthread_t(t_regular, "smthread_user_t"), 00105 _argc(ac), _argv(av), 00106 _device_name(NULL), 00107 _quota(0), 00108 _num_rec(0), 00109 _rec_size(0), 00110 _initialize_device(false), 00111 _options(NULL), 00112 _vid(1), 00113 retval(0) { } 00114 00115 ~smthread_user_t() { if(_options) delete _options; } 00116 00117 void run(); 00118 00119 // helpers for run() 00120 w_rc_t handle_options(); 00121 w_rc_t find_file_info(); 00122 w_rc_t create_the_file(); 00123 w_rc_t scan_the_file(); 00124 w_rc_t scan_the_root_index(); 00125 w_rc_t do_work(); 00126 w_rc_t do_init(); 00127 w_rc_t no_init(); 00128 00129 }; 00130 00131 /* 00132 * looks up file info in the root index 00133 */ 00134 w_rc_t 00135 smthread_user_t::find_file_info() 00136 { 00137 file_info_t info; 00138 W_DO(ssm->begin_xct()); 00139 00140 bool found; 00141 stid_t _root_iid; 00142 W_DO(ss_m::vol_root_index(_vid, _root_iid)); 00143 00144 smsize_t info_len = sizeof(info); 00145 const vec_t key_vec_tmp(file_info_t::key, strlen(file_info_t::key)); 00146 W_DO(ss_m::find_assoc(_root_iid, 00147 key_vec_tmp, 00148 &info, info_len, found)); 00149 if (!found) { 00150 cerr << "No file information found" <<endl; 00151 return RC(fcASSERT); 00152 } else { 00153 cerr << " found assoc " 00154 << file_info_t::key << " --> " << info << endl; 00155 } 00156 00157 W_DO(ssm->commit_xct()); 00158 00159 _start_rid = info.first_rid; 00160 _fid = info.fid; 00161 _rec_size = info.rec_size; 00162 _num_rec = info.num_rec; 00163 return RCOK; 00164 } 00165 00166 /* 00167 * This function either formats a new device and creates a 00168 * volume on it, or mounts an already existing device and 00169 * returns the ID of the volume on it. 00170 * 00171 * It's borrowed from elsewhere; it can handle mounting 00172 * an already existing device, even though in this main program 00173 * we don't ever do that. 00174 */ 00175 rc_t 00176 smthread_user_t::create_the_file() 00177 { 00178 file_info_t info; // will be made persistent in the 00179 // volume root index. 00180 00181 // create and fill file to scan 00182 cout << "Creating a file with " << _num_rec 00183 << " records of size " << _rec_size << endl; 00184 W_DO(ssm->begin_xct()); 00185 00186 // Create the file. Stuff its fid in the persistent file_info 00187 W_DO(ssm->create_file(_vid, info.fid, smlevel_3::t_regular)); 00188 rid_t rid; 00189 00190 _rec_size -= align(sizeof(int)); 00191 00192 /// each record will have its ordinal number in the header 00193 /// and zeros for data 00194 00195 char* dummy = new char[_rec_size]; 00196 memset(dummy, '\0', _rec_size); 00197 vec_t data(dummy, _rec_size); 00198 00199 for(int j=0; j < _num_rec; j++) 00200 { 00201 { 00202 w_ostrstream o(dummy, _rec_size); 00203 o << "Record number " << j << ends; 00204 w_assert1(o.c_str() == dummy); 00205 } 00206 // header contains record # 00207 int i = j; 00208 const vec_t hdr(&i, sizeof(i)); 00209 W_COERCE(ssm->create_rec(info.fid, hdr, 00210 _rec_size, data, rid)); 00211 cout << "Creating rec " << j << endl; 00212 if (j == 0) { 00213 info.first_rid = rid; 00214 } 00215 } 00216 cout << "Created all. First rid " << info.first_rid << endl; 00217 delete [] dummy; 00218 info.num_rec = _num_rec; 00219 info.rec_size = _rec_size; 00220 00221 // record file info in the root index : this stores some 00222 // attributes of the file in general 00223 stid_t _root_iid; 00224 W_DO(ss_m::vol_root_index(_vid, _root_iid)); 00225 00226 const vec_t key_vec_tmp(file_info_t::key, strlen(file_info_t::key)); 00227 const vec_t info_vec_tmp(&info, sizeof(info)); 00228 W_DO(ss_m::create_assoc(_root_iid, 00229 key_vec_tmp, 00230 info_vec_tmp)); 00231 cerr << "Creating assoc " 00232 << file_info_t::key << " --> " << info << endl; 00233 W_DO(ssm->commit_xct()); 00234 return RCOK; 00235 } 00236 00237 rc_t 00238 smthread_user_t::scan_the_root_index() 00239 { 00240 W_DO(ssm->begin_xct()); 00241 stid_t _root_iid; 00242 W_DO(ss_m::vol_root_index(_vid, _root_iid)); 00243 cout << "Scanning index " << _root_iid << endl; 00244 scan_index_i scan(_root_iid, 00245 scan_index_i::ge, vec_t::neg_inf, 00246 scan_index_i::le, vec_t::pos_inf, false, 00247 ss_m::t_cc_kvl); 00248 bool eof(false); 00249 int i(0); 00250 smsize_t klen(0); 00251 smsize_t elen(0); 00252 #define MAXKEYSIZE 100 00253 char * keybuf[MAXKEYSIZE]; 00254 file_info_t info; 00255 00256 do { 00257 w_rc_t rc = scan.next(eof); 00258 if(rc.is_error()) { 00259 cerr << "Error getting next: " << rc << endl; 00260 retval = rc.err_num(); 00261 return rc; 00262 } 00263 if(eof) break; 00264 00265 // get the key len and element len 00266 W_DO(scan.curr(NULL, klen, NULL, elen)); 00267 // Create vectors for the given lengths. 00268 vec_t key(keybuf, klen); 00269 vec_t elem(&info, elen); 00270 // Get the key and element value 00271 W_DO(scan.curr(&key, klen, &elem, elen)); 00272 00273 cout << "Key " << keybuf << endl; 00274 cout << "Value " 00275 << " { fid " << info.fid 00276 << " first_rid " << info.first_rid 00277 << " #rec " << info.num_rec 00278 << " rec size " << info.rec_size << " }" 00279 << endl; 00280 i++; 00281 } while (!eof); 00282 W_DO(ssm->commit_xct()); 00283 return RCOK; 00284 } 00285 00286 rc_t 00287 smthread_user_t::scan_the_file() 00288 { 00289 cout << "Scanning file " << _fid << endl; 00290 W_DO(ssm->begin_xct()); 00291 00292 scan_file_i scan(_fid); 00293 pin_i* cursor(NULL); 00294 bool eof(false); 00295 int i(0); 00296 00297 do { 00298 w_rc_t rc = scan.next(cursor, 0, eof); 00299 if(rc.is_error()) { 00300 cerr << "Error getting next: " << rc << endl; 00301 retval = rc.err_num(); 00302 return rc; 00303 } 00304 if(eof) break; 00305 00306 cout << "Record " << i << "/" << _num_rec 00307 << " Rid " << cursor->rid() << endl; 00308 vec_t header (cursor->hdr(), cursor->hdr_size()); 00309 int hdrcontents; 00310 header.copy_to(&hdrcontents, sizeof(hdrcontents)); 00311 cout << "Record hdr " << hdrcontents << endl; 00312 00313 const char *body = cursor->body(); 00314 w_assert1(cursor->body_size() == _rec_size); 00315 cout << "Record body " << body << endl; 00316 i++; 00317 } while (!eof); 00318 w_assert1(i == _num_rec); 00319 00320 W_DO(ssm->commit_xct()); 00321 return RCOK; 00322 } 00323 00324 rc_t 00325 smthread_user_t::do_init() 00326 { 00327 cout << "-i: Initialize " << endl; 00328 00329 { 00330 devid_t devid; 00331 cout << "Formatting device: " << _device_name 00332 << " with a " << _quota << "KB quota ..." << endl; 00333 W_DO(ssm->format_dev(_device_name, _quota, true)); 00334 00335 cout << "Mounting device: " << _device_name << endl; 00336 // mount the new device 00337 u_int vol_cnt; 00338 W_DO(ssm->mount_dev(_device_name, vol_cnt, devid)); 00339 00340 cout << "Mounted device: " << _device_name 00341 << " volume count " << vol_cnt 00342 << " device " << devid 00343 << endl; 00344 00345 // generate a volume ID for the new volume we are about to 00346 // create on the device 00347 cout << "Generating new lvid: " << endl; 00348 W_DO(ssm->generate_new_lvid(_lvid)); 00349 cout << "Generated lvid " << _lvid << endl; 00350 00351 // create the new volume 00352 cout << "Creating a new volume on the device" << endl; 00353 cout << " with a " << _quota << "KB quota ..." << endl; 00354 00355 W_DO(ssm->create_vol(_device_name, _lvid, _quota, false, _vid)); 00356 cout << " with local handle(phys volid) " << _vid << endl; 00357 00358 } 00359 00360 W_DO(create_the_file()); 00361 return RCOK; 00362 } 00363 00364 rc_t 00365 smthread_user_t::no_init() 00366 { 00367 cout << "Using already-existing device: " << _device_name << endl; 00368 // mount already existing device 00369 devid_t devid; 00370 u_int vol_cnt; 00371 w_rc_t rc = ssm->mount_dev(_device_name, vol_cnt, devid); 00372 if (rc.is_error()) { 00373 cerr << "Error: could not mount device: " 00374 << _device_name << endl; 00375 cerr << " Did you forget to run the server with -i?" 00376 << endl; 00377 return rc; 00378 } 00379 00380 // find ID of the volume on the device 00381 lvid_t* lvid_list; 00382 u_int lvid_cnt; 00383 W_DO(ssm->list_volumes(_device_name, lvid_list, lvid_cnt)); 00384 if (lvid_cnt == 0) { 00385 cerr << "Error, device has no volumes" << endl; 00386 exit(1); 00387 } 00388 _lvid = lvid_list[0]; 00389 delete [] lvid_list; 00390 00391 W_COERCE(find_file_info()); 00392 W_COERCE(scan_the_root_index()); 00393 W_DO(scan_the_file()); 00394 return RCOK; 00395 } 00396 00397 rc_t 00398 smthread_user_t::do_work() 00399 { 00400 if (_initialize_device) W_DO(do_init()); 00401 else W_DO(no_init()); 00402 return RCOK; 00403 } 00404 00405 /**\defgroup EGOPTIONS Example of setting up options. 00406 * This method creates configuration options, starts up 00407 * the storage manager, 00408 */ 00409 w_rc_t smthread_user_t::handle_options() 00410 { 00411 option_t* opt_device_name = 0; 00412 option_t* opt_device_quota = 0; 00413 option_t* opt_num_rec = 0; 00414 00415 cout << "Processing configuration options ..." << endl; 00416 00417 // Create an option group for my options. 00418 // I use a 3-level naming scheme: 00419 // executable-name.server.option-name 00420 // Thus, the file will contain lines like this: 00421 // create_rec.server.device_name : /tmp/example/device 00422 // *.server.device_name : /tmp/example/device 00423 // create_rec.*.device_name : /tmp/example/device 00424 // 00425 const int option_level_cnt = 3; 00426 00427 _options = new option_group_t (option_level_cnt); 00428 if(!_options) { 00429 cerr << "Out of memory: could not allocate from heap." << 00430 endl; 00431 retval = 1; 00432 return RC(fcINTERNAL); 00433 } 00434 option_group_t &options(*_options); 00435 00436 W_COERCE(options.add_option("device_name", "device/file name", 00437 NULL, "device containg volume holding file to scan", 00438 true, option_t::set_value_charstr, 00439 opt_device_name)); 00440 00441 W_COERCE(options.add_option("device_quota", "# > 1000", 00442 "2000", "quota for device", 00443 false, option_t::set_value_long, 00444 opt_device_quota)); 00445 00446 // Default number of records to create is 1. 00447 W_COERCE(options.add_option("num_rec", "# > 0", 00448 "1", "number of records in file", 00449 true, option_t::set_value_long, 00450 opt_num_rec)); 00451 00452 // Have the SSM add its options to my group. 00453 W_COERCE(ss_m::setup_options(&options)); 00454 00455 cout << "Finding configuration option settings." << endl; 00456 00457 w_rc_t rc = init_config_options(options, "server", _argc, _argv); 00458 if (rc.is_error()) { 00459 usage(options); 00460 retval = 1; 00461 return rc; 00462 } 00463 cout << "Processing command line." << endl; 00464 00465 // Process the command line: looking for the "-h" flag 00466 int option; 00467 while ((option = getopt(_argc, _argv, "hi")) != -1) { 00468 switch (option) { 00469 case 'i' : 00470 _initialize_device = true; 00471 break; 00472 00473 case 'h' : 00474 usage(options); 00475 break; 00476 00477 default: 00478 usage(options); 00479 retval = 1; 00480 return RC(fcNOTIMPLEMENTED); 00481 break; 00482 } 00483 } 00484 { 00485 cout << "Checking for required options..."; 00486 /* check that all required options have been set */ 00487 w_ostrstream err_stream; 00488 w_rc_t rc = options.check_required(&err_stream); 00489 if (rc.is_error()) { 00490 cerr << "These required options are not set:" << endl; 00491 cerr << err_stream.c_str() << endl; 00492 return rc; 00493 } 00494 cout << "Options OK; values are: { " << endl; 00495 options.print_values(false, cout); 00496 cout << "} end list of options values. " << endl; 00497 } 00498 00499 // Grab the options values for later use by run() 00500 _device_name = opt_device_name->value(); 00501 _quota = strtol(opt_device_quota->value(), 0, 0); 00502 _num_rec = strtol(opt_num_rec->value(), 0, 0); 00503 00504 return RCOK; 00505 } 00506 00507 void smthread_user_t::run() 00508 { 00509 w_rc_t rc = handle_options(); 00510 if(rc.is_error()) { 00511 retval = 1; 00512 return; 00513 } 00514 00515 // Now start a storage manager. 00516 cout << "Starting SSM and performing recovery ..." << endl; 00517 ssm = new ss_m(); 00518 if (!ssm) { 00519 cerr << "Error: Out of memory for ss_m" << endl; 00520 retval = 1; 00521 return; 00522 } 00523 00524 cout << "Getting SSM config info for record size ..." << endl; 00525 00526 sm_config_info_t config_info; 00527 W_COERCE(ss_m::config_info(config_info)); 00528 _rec_size = config_info.max_small_rec; // minus a header 00529 00530 // Subroutine to set up the device and volume and 00531 // create the num_rec records of rec_size. 00532 rc = do_work(); 00533 00534 if (rc.is_error()) { 00535 cerr << "Could not set up device/volume due to: " << endl; 00536 cerr << rc << endl; 00537 delete ssm; 00538 rc = RCOK; // force deletion of w_error_t info hanging off rc 00539 // otherwise a leak for w_error_t will be reported 00540 retval = 1; 00541 if(rc.is_error()) 00542 W_COERCE(rc); // avoid error not checked. 00543 return; 00544 } 00545 00546 sm_stats_info_t stats; 00547 W_COERCE(ss_m::gather_stats(stats)); 00548 cout << " SM Statistics : " << endl 00549 << stats << endl; 00550 00551 // Clean up and shut down 00552 cout << "\nShutting down SSM ..." << endl; 00553 delete ssm; 00554 00555 cout << "Finished!" << endl; 00556 00557 return; 00558 } 00559 00560 // This was copied from file_scan so it has lots of extra junk 00561 int 00562 main(int argc, char* argv[]) 00563 { 00564 smthread_user_t *smtu = new smthread_user_t(argc, argv); 00565 if (!smtu) 00566 W_FATAL(fcOUTOFMEMORY); 00567 00568 w_rc_t e = smtu->fork(); 00569 if(e.is_error()) { 00570 cerr << "error forking thread: " << e <<endl; 00571 return 1; 00572 } 00573 e = smtu->join(); 00574 if(e.is_error()) { 00575 cerr << "error forking thread: " << e <<endl; 00576 return 1; 00577 } 00578 00579 int rv = smtu->retval; 00580 delete smtu; 00581 00582 return rv; 00583 } 00584