Index: build.c =================================================================== RCS file: /cvsroot/mozilla/db/sqlite3/src/build.c,v retrieving revision 1.4.8.3 diff -u -u -8 -p -r1.4.8.3 build.c --- build.c 22 Feb 2006 20:56:28 -0000 1.4.8.3 +++ build.c 11 Apr 2006 17:28:51 -0000 @@ -22,16 +22,19 @@ ** COMMIT ** ROLLBACK ** ** $Id: preload-cache.patch,v 1.1.2.1 2006/04/11 17:33:53 brettw%gmail.com Exp $ */ #include "sqliteInt.h" #include +#include "pager.h" +#include "btree.h" + /* ** This routine is called when a new SQL statement is beginning to ** be parsed. Initialize the pParse structure as needed. */ void sqlite3BeginParse(Parse *pParse, int explainFlag){ pParse->explain = explainFlag; pParse->nVar = 0; } @@ -3214,8 +3217,34 @@ KeyInfo *sqlite3IndexKeyinfo(Parse *pPar } if( pParse->nErr ){ sqliteFree(pKey); pKey = 0; } return pKey; } + + +/* See declaration in sqlite3.h for information */ +int sqlite3Preload(sqlite3* db) +{ + Pager* pPager; + Btree* pBt; + int rc; + int i; + int dbsLoaded = 0; + + for (i = 0; i < db->nDb; i ++) { + pBt = db->aDb[i].pBt; + if (! pBt) + continue; + pPager = sqlite3BtreePager(pBt); + if (pPager) { + rc = sqlite3pager_loadall(pPager); + if (rc == SQLITE_OK) + dbsLoaded ++; + } + } + if (dbsLoaded == 0) + return SQLITE_ERROR; + return SQLITE_OK; +} Index: pager.c =================================================================== RCS file: /cvsroot/mozilla/db/sqlite3/src/pager.c,v retrieving revision 1.3.8.3 diff -u -u -8 -p -r1.3.8.3 pager.c --- pager.c 22 Feb 2006 20:56:28 -0000 1.3.8.3 +++ pager.c 11 Apr 2006 17:28:51 -0000 @@ -2521,16 +2521,30 @@ int sqlite3pager_release_memory(int nReq ** to find a page in the in-memory cache first. If the page is not already ** in memory, this routine goes to disk to read it in whereas _lookup() ** just returns 0. This routine acquires a read-lock the first time it ** has to go to disk, and could also playback an old journal if necessary. ** Since _lookup() never goes to disk, it never has to deal with locks ** or journal files. */ int sqlite3pager_get(Pager *pPager, Pgno pgno, void **ppPage){ + /* This just passes through to our modified version with a NULL data pointer */ + return sqlite3pager_get2(pPager, pgno, ppPage, 0); +} + + +/* +** This is an internal version of pager_get that takes an extra parameter of +** data to use to fill the page with. This allows more efficient filling for +** preloaded data. If this extra parameter is NULL, we'll go to the file. +** +** See sqlite3pager_loadall which uses this function. +*/ +int sqlite3pager_get2(Pager *pPager, Pgno pgno, void **ppPage, + unsigned char* pDataToFill) { PgHdr *pPg; int rc; /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page ** number greater than this, or zero, is requested. */ if( pgno>PAGER_MAX_PGNO || pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ return SQLITE_CORRUPT_BKPT; @@ -2676,39 +2690,46 @@ int sqlite3pager_get(Pager *pPager, Pgno } /* Populate the page with data, either by reading from the database ** file, or by setting the entire page to zero. */ if( sqlite3pager_pagecount(pPager)<(int)pgno || MEMDB ){ memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); }else{ - assert( MEMDB==0 ); - rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize); - if( rc==SQLITE_OK ){ - rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg), - pPager->pageSize); - } - TRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno); - CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3); - if( rc!=SQLITE_OK ){ - i64 fileSize; - int rc2 = sqlite3OsFileSize(pPager->fd, &fileSize); - if( rc2!=SQLITE_OK || fileSize>=pgno*pPager->pageSize ){ - /* An IO error occured in one of the the sqlite3OsSeek() or - ** sqlite3OsRead() calls above. */ - pPg->pgno = 0; - sqlite3pager_unref(PGHDR_TO_DATA(pPg)); - return rc; + if (pDataToFill) { + /* Just copy from the given memory */ + memcpy(PGHDR_TO_DATA(pPg), pDataToFill, pPager->pageSize); + CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3); + } else { + /* Load from disk */ + assert( MEMDB==0 ); + rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize); + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg), + pPager->pageSize); + } + TRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno); + CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3); + if( rc!=SQLITE_OK ){ + i64 fileSize; + int rc2 = sqlite3OsFileSize(pPager->fd, &fileSize); + if( rc2!=SQLITE_OK || fileSize>=pgno*pPager->pageSize ){ + /* An IO error occured in one of the the sqlite3OsSeek() or + ** sqlite3OsRead() calls above. */ + pPg->pgno = 0; + sqlite3pager_unref(PGHDR_TO_DATA(pPg)); + return rc; + }else{ + clear_simulated_io_error(); + memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); + } }else{ - clear_simulated_io_error(); - memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); + TEST_INCR(pPager->nRead); } - }else{ - TEST_INCR(pPager->nRead); } } /* Link the page into the page hash table */ h = pager_hash(pgno); pPg->pNextHash = pPager->aHash[h]; pPager->aHash[h] = pPg; if( pPg->pNextHash ){ @@ -3773,16 +3794,80 @@ int sqlite3pager_movepage(Pager *pPager, DATA_TO_PGHDR(pNeedSync)->dirty = 1; sqlite3pager_unref(pNeedSync); } return SQLITE_OK; } #endif + +/** + * Addition for Mozilla: This will attempt to populate the database cache with + * the first N bytes of the file, where N is the total size of the cache. + * Because we can load this as one chunk from the disk, this is much faster + * than loading a subset of the pages one at a time in random order. + * + * The pager must be initialized before this function is called. This means a + * statement must be open that has initialized the pager and is keeping the + * cache in memory. + */ +int sqlite3pager_loadall(Pager* pPager) +{ + int i; + int rc; + int loadSize; + int loadPages; + unsigned char* fileData; + + if (pPager->dbSize < 0 || pPager->pageSize < 0) { + /* pager not initialized, this means a statement is not open */ + return SQLITE_MISUSE; + } + + /* compute sizes */ + if (pPager->mxPage < pPager->dbSize) + loadPages = pPager->mxPage; + else + loadPages = pPager->dbSize; + loadSize = loadPages * pPager->pageSize; + + rc = sqlite3OsSeek(pPager->fd, 0); + if (rc != SQLITE_OK) + return rc; + + /* load the file as one chunk */ + fileData = sqliteMallocRaw(loadSize); + if (! fileData) + return SQLITE_NOMEM; + rc = sqlite3OsRead(pPager->fd, fileData, loadSize); + if (rc != SQLITE_OK) { + sqliteFree(fileData); + return rc; + } + + /* Copy the data to each page. Note that the page numbers we pass to _get + * are one-based, 0 is a marker for no page. We also need to check that we + * haven't loaded more pages than the cache can hold total. There may have + * already been a few pages loaded before, so we may fill the cache before + * loading all of the pages we want to. + */ + for (i = 1; i <= loadPages && pPager->nPage < pPager->mxPage; i ++) { + void *pPage; + rc = sqlite3pager_get2(pPager, 1, &pPage, + &fileData[(i-1)*(i64)pPager->pageSize]); + if (rc != SQLITE_OK) + break; + sqlite3pager_unref(pPage); + } + sqliteFree(fileData); + return SQLITE_OK; +} + + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Return the current state of the file lock for the given pager. ** The return value is one of NO_LOCK, SHARED_LOCK, RESERVED_LOCK, ** PENDING_LOCK, or EXCLUSIVE_LOCK. */ int sqlite3pager_lockstate(Pager *pPager){ return sqlite3OsLockState(pPager->fd); Index: pager.h =================================================================== RCS file: /cvsroot/mozilla/db/sqlite3/src/pager.h,v retrieving revision 1.2.8.3 diff -u -u -8 -p -r1.2.8.3 pager.h --- pager.h 22 Feb 2006 20:56:28 -0000 1.2.8.3 +++ pager.h 11 Apr 2006 17:28:51 -0000 @@ -104,16 +104,17 @@ const char *sqlite3pager_filename(Pager* const char *sqlite3pager_dirname(Pager*); const char *sqlite3pager_journalname(Pager*); int sqlite3pager_nosync(Pager*); int sqlite3pager_rename(Pager*, const char *zNewName); void sqlite3pager_set_codec(Pager*,void(*)(void*,void*,Pgno,int),void*); int sqlite3pager_movepage(Pager*,void*,Pgno); int sqlite3pager_reset(Pager*); int sqlite3pager_release_memory(int); +int sqlite3pager_loadall(Pager*); #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) int sqlite3pager_lockstate(Pager*); #endif #ifdef SQLITE_TEST void sqlite3pager_refdump(Pager*); int pager3_refinfo_enable; Index: sqlite3.h =================================================================== RCS file: /cvsroot/mozilla/db/sqlite3/src/sqlite3.h,v retrieving revision 1.2.2.4 diff -u -u -8 -p -r1.2.2.4 sqlite3.h --- sqlite3.h 22 Feb 2006 20:56:28 -0000 1.2.2.4 +++ sqlite3.h 11 Apr 2006 17:28:51 -0000 @@ -1482,12 +1482,25 @@ int sqlite3_table_column_metadata( */ int sqlite3_bind_parameter_indexes( sqlite3_stmt *pStmt, const char *zName, int **pIndexes ); void sqlite3_free_parameter_indexes(int *pIndexes); +/* +** Preload the databases into the pager cache, up to the maximum size of the +** pager cache. +** +** For a database to be loaded successfully, the pager must be active. That is, +** there must be an open statement on that database. See sqlite3pager_loadall +** +** There might be many databases attached to the given connection. We iterate +** them all and try to load them. If none are loadable successfully, we return +** an error. Otherwise, we return OK. +*/ +int sqlite3Preload(sqlite3* db); + #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif #endif