Index: include/internal/chewing-private.h =================================================================== --- include/internal/chewing-private.h (revision 885) +++ include/internal/chewing-private.h (working copy) @@ -42,6 +42,9 @@ typedef struct { uint16 phone_id; int phrase_id; int child_begin, child_end; +//#ifdef USE_BINARY_DATA_SQLITE + int fetched; +//#endif } TreeType; typedef struct { Index: include/internal/global-private.h =================================================================== --- include/internal/global-private.h (revision 885) +++ include/internal/global-private.h (working copy) @@ -13,6 +13,12 @@ #ifndef _CHEWING_GLOBAL_PRIVATE_H #define _CHEWING_GLOBAL_PRIVATE_H +#ifdef HAVE_CONFIG_H +#ifndef PACKAGE_NAME +#include "config.h" +#endif +#endif + #define PHONE_TREE_FILE "fonetree.dat" #define DICT_FILE "dict.dat" #define PH_INDEX_FILE "ph_index.dat" Index: src/tree.c =================================================================== --- src/tree.c (revision 885) +++ src/tree.c (working copy) @@ -31,6 +31,10 @@ #include "private.h" #include "plat_mmap.h" +#ifdef USE_BINARY_DATA_SQLITE +#include +#endif + #define INTERVAL_SIZE ( ( MAX_PHONE_SEQ_LEN + 1 ) * MAX_PHONE_SEQ_LEN / 2 ) typedef struct { @@ -55,10 +59,14 @@ typedef struct { } TreeDataType; #ifdef USE_BINARY_DATA -extern TreeType *tree; +TreeType *tree; static unsigned int tree_size = 0; -#else -extern TreeType tree[ TREE_SIZE ]; +#ifdef USE_BINARY_DATA_SQLITE +static sqlite3 *db = NULL; +static sqlite3_stmt *stFetch; +#endif +#else /* ! USE_BINARY_DATA */ +TreeType tree[ TREE_SIZE ]; #endif static int IsContain( IntervalType in1, IntervalType in2 ) @@ -98,6 +106,10 @@ static void TerminateTree() #ifdef USE_BINARY_DATA if ( tree ) free( tree ); +#ifdef USE_BINARY_DATA_SQLITE + if ( stFetch ) + sqlite3_finalize( stFetch ); +#endif #endif } @@ -106,15 +118,52 @@ void ReadTree( const char *prefix ) char filename[ 100 ]; #ifdef USE_BINARY_DATA +#ifdef USE_BINARY_DATA_SQLITE + int rc; + sqlite3_stmt *st; + char *buf; +#else /* ! USE_BINARY_DATA_SQLITE */ int infile; struct stat file_stat; -#else +#endif +#else /* ! USE_BINARY_DATA */ FILE *infile; int i; #endif sprintf( filename, "%s" PLAT_SEPARATOR "%s", prefix, PHONE_TREE_FILE ); #ifdef USE_BINARY_DATA +#ifdef USE_BINARY_DATA_SQLITE + rc = sqlite3_open( filename, &db ); + if ( rc ) { + DEBUG_OUT( "Can't open database: %s\n", + sqlite3_errmsg( db ) ); + sqlite3_close( db ); + /* FIXME: exception handling with database */ + return; + } + rc = sqlite3_prepare( db, + "SELECT id,phone_id,phrase_id,child_begin,child_end FROM tree WHERE id = ?", + -1 , &stFetch, + (const char **) &buf); + if ( rc != SQLITE_OK ) { + DEBUG_OUT( "SQL error (prepare SELECT): %d\n", rc ); + } + + rc = sqlite3_prepare( db, + "SELECT max(id) FROM tree" , + -1 , + &st , (const char **) &buf); + rc = sqlite3_step( st ); + if ( rc != SQLITE_ROW ) { + DEBUG_OUT( "SQL error: %d\n", rc ); + } + tree_size = sqlite3_column_int( st, 0 ) + 1; + tree = ALC( TreeType, tree_size ); + memset( tree, 0, tree_size * sizeof(TreeType) ); + + sqlite3_finalize( st ); +#else /* ! USE_BINARY_DATA_SQLITE */ infile = open( filename, O_RDONLY ); if ( infile == -1 ) @@ -127,7 +176,8 @@ void ReadTree( const char *prefix ) read( infile, tree, tree_size ); close( infile ); -#else +#endif +#else /* ! USE_BINARY_DATA */ infile = fopen( filename, "r" ); assert( infile ); for ( i = 0; i < TREE_SIZE; i++ ) { @@ -276,6 +326,23 @@ static int CheckChoose( return 0; } +#ifdef USE_BINARY_DATA_SQLITE +static int TreeFetchNode( int id ) +{ + if ( ! tree[ id ].fetched ) { + sqlite3_reset( stFetch ); + sqlite3_bind_int( stFetch, 1, id ); + sqlite3_step( stFetch ); + + tree[ id ].phone_id = sqlite3_column_int( stFetch, 1 ); + tree[ id ].phrase_id = sqlite3_column_int( stFetch, 2 ); + tree[ id ].child_begin = sqlite3_column_int( stFetch, 3 ); + tree[ id ].child_end = sqlite3_column_int( stFetch, 4 ); + tree[ id ].fetched = 1; + } +} +#endif + /** @brief search for the phrases have the same pronunciation.*/ /* if phoneSeq[a] ~ phoneSeq[b] is a phrase, then add an interval * from (a) to (b+1) */ @@ -285,6 +352,9 @@ int TreeFindPhrase( int begin, int end, tree_p = 0; for ( i = begin; i <= end; i++ ) { +#ifdef USE_BINARY_DATA_SQLITE + TreeFetchNode( tree_p ); +#endif for ( child = tree[ tree_p ].child_begin; child <= tree[ tree_p ].child_end; @@ -299,8 +369,13 @@ int TreeFindPhrase( int begin, int end, */ if ( child < 0 || child * sizeof(TreeType) > tree_size ) return -1; +#elif defined(USE_BINARY_DATA_SQLITE) + if ( child == -1 ) + continue; + if ( child * sizeof(TreeType) > tree_size ) + return -1; + TreeFetchNode( child ); #endif - if ( tree[ child ].phone_id == phoneSeq[ i ] ) break; } Index: src/tools/dat2bin.c =================================================================== --- src/tools/dat2bin.c (revision 885) +++ src/tools/dat2bin.c (working copy) @@ -36,7 +36,9 @@ #define USED_IN_DAT2BIN #include "sort_word.c" #include "sort_dic.c" +#ifndef USE_BINARY_DATA_SQLITE #include "maketree.c" +#endif #undef USED_IN_DAT2BIN int main( int argc, char* argv[] ) @@ -56,8 +58,10 @@ int main( int argc, char* argv[] ) printf("sort_dic...\n"); sort_dic(); +#ifndef USE_BINARY_DATA_SQLITE printf("maketree...\n"); maketree(); +#endif printf("sort_word...\n"); sort_word(); @@ -91,6 +95,9 @@ int main( int argc, char* argv[] ) } fclose( fi ); +#ifdef USE_BINARY_DATA_SQLITE + /* Do nothing */ +#else /* PHONE_TREE_FILE */ sprintf( filename, "%s" PLAT_SEPARATOR "%s", prefix, PHONE_TREE_FILE ); fi = fopen( filename, "r" ); @@ -113,6 +120,7 @@ int main( int argc, char* argv[] ) fclose( fo ); } fclose( fi ); +#endif /* PH_INDEX_FILE */ sprintf( filename, "%s" PLAT_SEPARATOR "%s", prefix, PH_INDEX_FILE ); Index: src/tools/Makefile.am =================================================================== --- src/tools/Makefile.am (revision 885) +++ src/tools/Makefile.am (working copy) @@ -2,7 +2,8 @@ INCLUDES = \ -I$(top_srcdir)/include \ -I$(top_srcdir)/include/internal \ -I$(top_srcdir)/src \ - -I$(top_srcdir)/src/porting_layer/include + -I$(top_srcdir)/src/porting_layer/include \ + $(SQLITE3_CFLAGS) noinst_PROGRAMS = sort_word sort_dic maketree $(DAT2BIN) @@ -19,3 +20,7 @@ sort_dic_LDADD = $(top_builddir)/src/co sort_dic_SOURCES = sort_dic.c maketree_SOURCES = maketree.c + +if ENABLE_BINARY_DATA_SQLITE +maketree_LDADD = $(SQLITE3_LIBS) +endif Index: src/tools/maketree.c =================================================================== --- src/tools/maketree.c (revision 885) +++ src/tools/maketree.c (working copy) @@ -5,7 +5,7 @@ * Lu-chuan Kung and Kang-pen Chen. * All rights reserved. * - * Copyright (c) 2004 + * Copyright (c) 2004, 2008 * libchewing Core Team. See ChangeLog for details. * * See the file "COPYING" for information on usage and redistribution @@ -33,6 +33,9 @@ #include #include "global.h" #include "global-private.h" +#ifdef USE_BINARY_DATA_SQLITE +#include +#endif /* defines @@ -194,14 +197,61 @@ void BFS2() { NODE *pNode; LISTNODE *pList; - FILE *output = fopen( PHONE_TREE_FILE, "w" ); - FILE *config = fopen( CHEWING_DEFINITION_FILE, "aw" ); + FILE *config; +#ifdef USE_BINARY_DATA_SQLITE + int rc; + int i; + sqlite3 *db; + sqlite3_stmt *st; + char *zErrMsg = 0; + char *buf; + + rc = sqlite3_open( PHONE_TREE_FILE, &db ); + if (rc) { + fprintf( stderr, "Can't open database: %s\n", sqlite3_errmsg( db ) ); + sqlite3_close( db ); + exit( 1 ); + } + + rc = sqlite3_exec( db, + "CREATE TABLE tree (id,phone_id,phrase_id,child_begin,child_end)", + NULL, NULL, &zErrMsg ); + if ( rc != SQLITE_OK ) { + fprintf( stderr, "A SQL error: %s\n", zErrMsg ); + } + rc = sqlite3_exec( db, + "CREATE INDEX tree_index ON tree (id)", + NULL, NULL, &zErrMsg ); + if ( rc != SQLITE_OK ) { + fprintf( stderr, "A SQL error: %s\n", zErrMsg ); + } + + rc = sqlite3_exec( db, + "CREATE INDEX tree_index ON tree (id)", + NULL, NULL, &zErrMsg ); + rc = sqlite3_prepare( db, + "INSERT INTO tree VALUES (?,?,?,?,?)", + -1 , &st , (const char **) &buf ); + if ( rc != SQLITE_OK ) { + fprintf( stderr, "B SQL error: %d\n", rc ); + } + + rc = sqlite3_exec( db, + "BEGIN", + NULL, NULL, &zErrMsg ); + if ( rc != SQLITE_OK ) { + fprintf( stderr, "C SQL error: %s\n", zErrMsg ); + } +#else /* ! USE_BINARY_DATA_SQLITE */ + FILE *output = fopen( PHONE_TREE_FILE, "w" ); if ( ! output ) { fprintf( stderr, "Error opening file " PHONE_TREE_FILE " for output.\n" ); exit( 1 ); } +#endif + config = fopen( CHEWING_DEFINITION_FILE, "aw" ); if ( ! config ) { fprintf( stderr, "Error opening file " CHEWING_DEFINITION_FILE " for output.\n" ); exit( 1 ); @@ -209,30 +259,65 @@ void BFS2() QueuePut( root ); tree_size = 0; + +#ifdef USE_BINARY_DATA_SQLITE + i = 0; +#endif while ( ! QueueEmpty() ) { pNode = QueueGet(); - + +#ifdef USE_BINARY_DATA_SQLITE + sqlite3_bind_int( st, 1, i++ ); + sqlite3_bind_int( st, 2, (int) pNode->key ); + sqlite3_bind_int( st, 3, pNode->phraseno ); +#else /* ! USE_BINARY_DATA_SQLITE */ fprintf( output, "%hu ", pNode->key ); fprintf( output, "%d ", pNode->phraseno ); +#endif /* compute the begin and end index */ pList = pNode->childList; - if( pList ) { + if ( pList ) { +#ifdef USE_BINARY_DATA_SQLITE + sqlite3_bind_int( st, 4, pList->pNode->nodeno ); +#else /* ! USE_BINARY_DATA_SQLITE */ fprintf( output, "%d ", pList->pNode->nodeno ); +#endif for ( ; pList->next; pList = pList->next ) { QueuePut( pList->pNode ); } QueuePut( pList->pNode ); +#ifdef USE_BINARY_DATA_SQLITE + sqlite3_bind_int( st, 5, pList->pNode->nodeno ); + } + else { + sqlite3_bind_int( st, 4, -1 ); + sqlite3_bind_int( st, 5, -1 ); + } +#else /* ! USE_BINARY_DATA_SQLITE */ fprintf( output, "%d\n", pList->pNode->nodeno ); } else fprintf( output, "-1 -1\n" ); +#endif tree_size++; +#ifdef USE_BINARY_DATA_SQLITE + sqlite3_step( st ); + sqlite3_reset( st ); +#endif } fprintf( config, "#define TREE_SIZE (%d)\n", tree_size ); +#ifdef USE_BINARY_DATA_SQLITE + rc = sqlite3_exec( db, "COMMIT", NULL, NULL, &zErrMsg ); + if ( rc!=SQLITE_OK ) { + fprintf( stderr, "D SQL error: %s\n", zErrMsg ); + } + sqlite3_close( db ); +#else /* ! USE_BINARY_DATA_SQLITE */ fclose( output ); fclose( config ); +#endif } #ifdef USED_IN_DAT2BIN Index: src/Makefile.am =================================================================== --- src/Makefile.am (revision 885) +++ src/Makefile.am (working copy) @@ -28,3 +28,9 @@ libchewing_la_LDFLAGS = \ -rpath $(libdir) \ -no-undefined \ -export-symbols-regex "^(chewing)_" + +if ENABLE_BINARY_DATA_SQLITE +INCLUDES += $(SQLITE3_CFLAGS) +libchewing_la_LDFLAGS += $(SQLITE3_LIBS) +endif + Index: src/choice.c =================================================================== --- src/choice.c (revision 885) +++ src/choice.c (working copy) @@ -25,18 +25,11 @@ #include "global.h" #include "dict-private.h" #include "char-private.h" -#include "tree-private.h" #include "chewingutil.h" #include "userphrase-private.h" #define CEIL_DIV( a, b ) ( ( a + b - 1 ) / b ) -#ifdef USE_BINARY_DATA -TreeType *tree = NULL; -#else -TreeType tree[ TREE_SIZE ]; -#endif - static void ChangeSelectIntervalAndBreakpoint( ChewingData *pgdata, int from, Index: configure.ac =================================================================== --- configure.ac (revision 885) +++ configure.ac (working copy) @@ -79,8 +79,13 @@ AC_FUNC_MALLOC AC_FUNC_MEMCMP AC_CHECK_FUNCS([memchr memmove memset mkdir strchr strstr]) +# optional modules +CHECK_REQUIRED_VERSION=0.9.4 +SQLITE_REQUIRED_VERSION=3.0 + # Checks for check unit test framework -PKG_CHECK_MODULES(CHECK, check >= 0.9.4, enable_check=yes, enble_check=no) +PKG_CHECK_MODULES(CHECK, check >= $CHECK_REQUIRED_VERSION, + enable_check=yes, enble_check=no) AM_CONDITIONAL(ENABLE_UNIT_TEST, test x$enable_check = "xyes") # Checks for ncursesw @@ -169,15 +174,38 @@ AC_ARG_ENABLE([binary-data], binary_data="no" ;; *) + binary_data="yes" ;; - esac], - [binary_data="yes"]) + esac],) if test x$binary_data = "xyes"; then AC_DEFINE(USE_BINARY_DATA, 1, [Experimental use of binary data]) fi -AC_SUBST(ENABLE_BINARY_DATA) AM_CONDITIONAL(ENABLE_BINARY_DATA, test x$binary_data = "xyes") +AC_ARG_ENABLE([sqlite], + [AS_HELP_STRING([--enable-sqlite], + [Experimental use of SQLite3 for binary data @<:@default=no@:>@])], + [case "${enableval}" in + yes) + binary_data_sqlite="yes" + ;; + *) + ;; + esac], + [binary_data_sqlite="no"]) +if test x$binary_data = "xyes" -a x$binary_data_sqlite = "xyes"; then + PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= $SQLITE_REQUIRED_VERSION]) + AC_SUBST([SQLITE3_CFLAGS]) + AC_SUBST([SQLITE3_LIBS]) + AC_DEFINE(USE_BINARY_DATA_SQLITE, 1, + [Experimental use of SQLite 3 for binary data]) +else + binary_data_sqlite="no" +fi +AM_CONDITIONAL(ENABLE_BINARY_DATA_SQLITE, + test x$binary_data_sqlite = "xyes") + + # Platform-dependent dnl What kind of system are we using? case $host_os in @@ -217,12 +245,18 @@ AC_OUTPUT([ Doxyfile ]) -AC_MSG_RESULT([ -Build options: - Version $PACKAGE_VERSION - Install prefix $prefix - Enable debug $LIBDEBUG - Enable binary data $binary_data - Build Unit Test $enable_check - Build TextUI sample $enable_ncursesw -]) +AC_OUTPUT + +echo +echo Build options: +echo " Version $PACKAGE_VERSION" +echo " Install prefix $prefix" +echo " Enable debug $LIBDEBUG" +echo " Enable binary data $binary_data" +if test x$binary_data = "xyes"; then +echo " +- SQLite support $binary_data_sqlite" +fi +echo " Build Unit Test $enable_check" +echo " Build TextUI sample $enable_ncursesw" +echo + Index: data/Makefile.am =================================================================== --- data/Makefile.am (revision 885) +++ data/Makefile.am (working copy) @@ -26,9 +26,19 @@ gendata: if ENABLE_BINARY_DATA convert_dat: $(tooldir)/dat2bin -gendata2: ch_index.dat_bin fonetree.dat_bin ph_index.dat_bin +GENERATED_DATA_LIST = ch_index.dat_bin ph_index.dat_bin +if ENABLE_BINARY_DATA_SQLITE +else +GENERATED_DATA_LIST += fonetree.dat_bin +endif +gendata2: $(GENERATED_DATA_LIST) -mv -f ch_index.dat_bin ch_index.dat +if ENABLE_BINARY_DATA_SQLITE + -rm -f fonetree.dat + $(tooldir)/maketree +else -mv -f fonetree.dat_bin fonetree.dat +endif -mv -f ph_index.dat_bin ph_index.dat else $(tooldir)/sort_word