[Lsb-messages] /var/www/bzr/lsb/devel/specdb r1254: Refactor specdb build/load/dump system.

Jeff Licquia licquia at linuxfoundation.org
Tue Jan 22 02:00:26 UTC 2013


------------------------------------------------------------
revno: 1254
committer: Jeff Licquia <licquia at linuxfoundation.org>
branch nick: specdb
timestamp: Mon 2013-01-21 21:00:26 -0500
message:
  Refactor specdb build/load/dump system.
  
  This was prompted by continual dump/restore issues being seen by Mats
  and me.  New features/benefits:
  
   - Individual tables can be dumped or restored.
  
   - Procedure for dumping/restoring is no longer repeated in multiple
     places; when changes need to be made, they can be made once.
  
   - Support for handling the strange behavior of LOAD DATA INFILE in
     MySQL.  Since LOAD DATA INFILE with --local-infile is treated
     differently than LOAD DATA LOCAL INFILE, we now support doing the
     latter via the LOCAL_INFILE_CMD makefile setting.
  
   - Related to the above, use DO_COPY to control whether to try and
     dump/restore in-place, or use a temporary location as an
     intermediate.
  
   - Use the table lists in the makefile, rather than looping over *.sql
     for lists of tables.
  
   - Get rid of embedded shell loops generally in favor of proper make
     dependencies.
  
   - Break out most tasks into separate make targets.  New targets:
     dbsetup, permissions, alias_detector, optimize_speconly, clean.
  
   - Break out permissions setting from the rest of the procedures, so
     they are not run by default.  This may mean that 'make permissions'
     may need to be run manually when setting up a new database, but
     prevents errors that occur when the DB user doesn't have GRANT
     rights (as is currently the case with the default security
     settings).
  
   - As a supporting set of rules, create a list of elements for use by
     the other rules, and clean it up with 'make clean'.
added:
  .bzrignore
modified:
  create_cache_tables_inits.sh
  makefile
-------------- next part --------------
=== added file '.bzrignore'
--- a/.bzrignore	1970-01-01 00:00:00 +0000
+++ b/.bzrignore	2013-01-22 02:00:26 +0000
@@ -0,0 +1,1 @@
+elements_list

=== modified file 'create_cache_tables_inits.sh'
--- a/create_cache_tables_inits.sh	2011-12-12 07:35:05 +0000
+++ b/create_cache_tables_inits.sh	2013-01-22 02:00:26 +0000
@@ -13,9 +13,9 @@
 LC_ALL=C cut cache_RLibRIntMapping.init -f2 | LC_ALL=C uniq >cache_RIntNames.init
 LC_ALL=C cut RawInterface.init -f2,3,5 | LC_ALL=C sort -u --ignore-case | LC_ALL=C nl -w1 >cache_RIntCaseInsensitiveNames.init
 
-mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA INFILE '$PWD/cache_RIntNames.init' into table cache_RIntNames fields enclosed by '\''; optimize table cache_RIntNames"
-mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA INFILE '$PWD/cache_RLibRIntMapping.init' into table cache_RLibRIntMapping fields enclosed by '\''; optimize table cache_RLibRIntMapping"
-mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA INFILE '$PWD/cache_RIntCaseInsensitiveNames.init' into table cache_RIntCaseInsensitiveNames fields enclosed by '\''; optimize table cache_RIntCaseInsensitiveNames"
+mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA $LOCALCMD INFILE '$PWD/cache_RIntNames.init' into table cache_RIntNames fields enclosed by '\''; optimize table cache_RIntNames"
+mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA $LOCALCMD INFILE '$PWD/cache_RLibRIntMapping.init' into table cache_RLibRIntMapping fields enclosed by '\''; optimize table cache_RLibRIntMapping"
+mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA $LOCALCMD INFILE '$PWD/cache_RIntCaseInsensitiveNames.init' into table cache_RIntCaseInsensitiveNames fields enclosed by '\''; optimize table cache_RIntCaseInsensitiveNames"
 mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "UPDATE cache_RIntCaseInsensitiveNames SET RIunmangled=NULL WHERE RIunmangled='' "
 
 # For each LSB version, create a table with interfaces included in it (at least at one architecture)

=== modified file 'makefile'
--- a/makefile	2012-11-01 02:02:50 +0000
+++ b/makefile	2013-01-22 02:00:26 +0000
@@ -7,10 +7,16 @@
 
 DBOPTS=-h $$LSBDBHOST -u $$LSBUSER --password=$$LSBDBPASSWD
 DUMPOPTS=--quote-names --extended-insert=false --triggers=FALSE --skip-dump-date --skip-comments
-# Add '--local-infile' here if there are any problems with direct access to files
-FILEOPTS=
-# If TMPLSBDB environment variable is set, it will be used for temp database name;
-# otherwise we'll use "${LSBDB}_tmp"
+# Sometimes LOAD DATA INFILE doesn't work, but ...LOCAL INFILE does.
+# If this is your case, set this to 'local'.
+LOCAL_INFILE_CMD=
+# If DO_COPY is 'yes', then we copy files being loaded with LOAD DATA INFILE
+# to a common temporary location first.
+DO_COPY=no
+# Use this temporary location if needed.
+TMPDIR=/tmp
+# If TMPLSBDB environment variable is set, it will be used for temp database
+# name; otherwise we'll use "${LSBDB}_tmp"
 TMPLSBDB=$${TMPLSBDB=${LSBDB}}_tmp
 
 ELEMENTS=AbiApi AbiMacro ArchClass ArchConst ArchDE ArchES ArchInt \
@@ -29,42 +35,63 @@
 	JavaInterface RILMBuiltin RLibDeps RLibLink RLibRClass RLibRInt RawClass RawCommand RawILModule \
 	RawInterface RawLibSoname RawLibrary WeakSymbol
 
+ifeq (yes,$(DO_COPY))
+LOAD_LOCATION=$(TMPDIR)
+else
+LOAD_LOCATION=$$PWD
+endif
+
+# These variables hold the targets for each thing that needs to be done
+# in the form 'tablename_job'.  Example: 'Type_dump' would dump the Type
+# table.  Most of the work this makefile does is based on these rules.
+
+ELEMENTS_DUMP = $(shell echo $(ELEMENTS) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_dump"; done))
+COMMUNITY_DUMP = $(shell echo $(COMMUNITY) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_dump"; done))
+APP_TABLES_DUMP = $(shell echo $(APP_TABLES) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_dump"; done))
+ELEMENTS_RESTORE = $(shell echo $(ELEMENTS) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_restore"; done))
+COMMUNITY_RESTORE = $(shell echo $(COMMUNITY) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_restore"; done))
+APP_TABLES_RESTORE = $(shell echo $(APP_TABLES) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_restore"; done))
+
 all:
 	@echo "Please specify dump or restore (or variants dumpall, restoreall, dump_apps, restore_apps)"
 
+# This is used to distinguish which dump and restore commands to use
+# for the data of a particular table.
+elements_list:
+	echo $(ELEMENTS) > $@
+
+%_dump: elements_list
+	mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $* | grep -v 'Server version' |grep -v 'character_set_client' > $*.sql
+	if grep -q $* elements_list; then \
+	  mysqldump $(DBOPTS) $(DUMPOPTS) $$LSBDB $* | LC_ALL=C grep INSERT > $*.init; \
+	else \
+	  rm -f "$$PWD/$*.init" && mysql $(DBOPTS) $$LSBDB -e "select * from $* into outfile '$$PWD/$*.init'"; \
+	fi
+
+%_restore: elements_list
+	@echo $*
+	@mysql $(DBOPTS) $$LSBDB < $*.sql
+	@if grep -q $* elements_list; then \
+	  mysql $(DBOPTS) $$LSBDB < $*.init; \
+	else \
+	  cp $*.init $(TMPDIR); \
+	  chmod 644 $(TMPDIR)/$*.init; \
+	  mysql $(DBOPTS) $$LSBDB -e "load data $(LOCAL_INFILE_CMD) infile '$(LOAD_LOCATION)/$*.init' into table $*"; \
+	  rm -f $(TMPDIR)/$*.init; \
+	fi
+
 # dump the "source code" tables
-dump::
-	for table in $(ELEMENTS) ; \
-	do \
-		set +e; \
-		echo $$table; \
-		mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | grep -v 'Server version' |grep -v 'character_set_client' >$$table.sql;\
-		mysqldump $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | LC_ALL=C grep INSERT >$$table.init;\
-	done
+dump: $(ELEMENTS_DUMP)
 
 # dump everything: the "source code" tables + the "community" tables
 # if the community tables are populated, this will be big!
-dumpall::
-	for table in `mysql $(DBOPTS) -e "SHOW TABLES" $$LSBDB | grep -v Tables | grep -v cache_` ;\
-	do \
-		set +e; \
-		echo $$table; \
-		mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | grep -v 'Server version' |grep -v 'character_set_client' >$$table.sql;\
-		case "$(ELEMENTS)" in \
-			*"$$table"*) mysqldump $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | LC_ALL=C grep INSERT >$$table.init ;; \
-			*) rm -f "$$PWD/$$table.init" && mysql $(DBOPTS) $$LSBDB -e "select * from $$table into outfile '$$PWD/$$table.init'" ;; \
-		esac;\
-	done
+dumpall: dump $(COMMUNITY_DUMP)
 
-restore::
-	for table in $(ELEMENTS) ; \
-	do \
-		set +e; \
-		echo $$table; \
-		mysql $(DBOPTS) $$LSBDB <$$table.sql; \
-		mysql $(DBOPTS) $$LSBDB <$$table.init; \
-		mysql $(DBOPTS) $$LSBDB -e "OPTIMIZE TABLE $$table"; \
-	done
+dbsetup::
+	mysql $(DBOPTS) -e "drop database if exists $$LSBDB"
+	mysql $(DBOPTS) -e "drop database if exists $(TMPLSBDB)"
+	@mysqladmin $(DBOPTS) create $$LSBDB
+	@mysqladmin $(DBOPTS) create $(TMPLSBDB)
 
 community_check::
 	@ret_code=0; \
@@ -82,34 +109,25 @@
 	fi; \
 	exit $$ret_code
 
-restoreall:: community_check
-	mysql $(DBOPTS) -e "drop database if exists $$LSBDB"
-	mysql $(DBOPTS) -e "drop database if exists $(TMPLSBDB)"
-	@mysqladmin $(DBOPTS) create $$LSBDB
-	@mysqladmin $(DBOPTS) create $(TMPLSBDB)
-	#mysql $(DBOPTS) $$LSBDB <setupdb.sql;
-#	sleep 5
-	LC_ALL=C $(SHELL) -c 'for table in [A-Z]*sql ; \
-	do \
-		set +e; \
-		table=`basename $$table .sql`; \
-		echo $$table; \
-		mysql $(DBOPTS) $$LSBDB <$$table.sql; \
-		case "$(ELEMENTS)" in \
-		        *"$$table"*) mysql $(DBOPTS) $$LSBDB <$$table.init ;; \
-		        *) cp $${table}.init /tmp; chmod 644 /tmp/$${table}.init; mysql $(DBOPTS) $(FILEOPTS) $$LSBDB -e "load data infile \"/tmp/$${table}.init\" into table $$table"; rm /tmp/$${table}.init ;; \
-		esac;\
-	done'
-	FILEOPTS=$(FILEOPTS) ./create_cache_tables_inits.sh
-	mysql $(DBOPTS) $$LSBDB <create_cache_tables.sql;
+alias_detector::
 	mysql $(DBOPTS) $$LSBDB <create_alias_detector.sql;
-	mysql $(DBOPTS) $$LSBDB <create_stored_procs.sql
-	rm -f cache*.init
+
+optimize::
 	LC_ALL=C $(SHELL) -c 'for table in [A-Z]*sql ; \
 	do \
-		table=`basename $$table .sql`; \
-		mysql $(DBOPTS) $$LSBDB -e "SET SESSION myisam_sort_buffer_size = 30 * 1024 * 1024; OPTIMIZE TABLE $$table"; \
+	    table=`basename $$table .sql`; \
+	    mysql $(DBOPTS) $$LSBDB -e "SET SESSION myisam_sort_buffer_size = 30 * 1024 * 1024; OPTIMIZE TABLE $$table"; \
 	done'
+
+optimize_speconly::
+	for table in $(ELEMENTS) ; \
+	do \
+	        set +e; \
+	        echo $$table; \
+	        mysql $(DBOPTS) $$LSBDB -e "OPTIMIZE TABLE $$table"; \
+	done
+
+permissions::
 	mysql $(DBOPTS) $$LSBDB <dbperms.sql;
 	mysql $(DBOPTS) $(TMPLSBDB) <tmpdbperms.sql;
 
@@ -117,54 +135,21 @@
 # then call the 'cache' rule
 
 cache::
-#	FILEOPTS=$(FILEOPTS) ./create_cache_tables_inits.sh
+	LOCALCMD=$(LOCAL_INFILE_CMD) ./create_cache_tables_inits.sh
 	mysql $(DBOPTS) $$LSBDB <create_cache_tables.sql;
 	mysql $(DBOPTS) $$LSBDB <create_stored_procs.sql
 	rm -f cache*.init
-	LC_ALL=C $(SHELL) -c 'for table in [A-Z]*sql ; \
-	do \
-	    table=`basename $$table .sql`; \
-	    mysql $(DBOPTS) $$LSBDB -e "SET SESSION myisam_sort_buffer_size = 30 * 1024 * 1024; OPTIMIZE TABLE $$table"; \
-	done'
-	mysql $(DBOPTS) $$LSBDB <dbperms.sql;
-	mysql $(DBOPTS) $(TMPLSBDB) <tmpdbperms.sql;
-
+
+restore: $(ELEMENTS_RESTORE) optimize_speconly
+
+restoreall:: community_check dbsetup $(ELEMENTS_RESTORE) $(COMMUNITY_RESTORE) \
+  alias_detector cache optimize
 
 # rules to process application data only
 
-restore_apps::
-	for table in $(APP_TABLES) ; \
-	do \
-		set +e; \
-		echo $$table; \
-	        mysql $(DBOPTS) $$LSBDB <$$table.sql; \
-	        cp $$table.init /tmp; \
-	        chmod 644 /tmp/$$table.init; \
-		mysql $(DBOPTS) $(FILEOPTS) $$LSBDB -e "load data infile '$$PWD/$$table.init' into table $$table";\
-	        rm /tmp/$$table.init; \
-	done
-	FILEOPTS=$(FILEOPTS) ./create_cache_tables_inits.sh
-	mysql $(DBOPTS) $$LSBDB <create_cache_tables.sql;
-	mysql $(DBOPTS) $$LSBDB <create_stored_procs.sql 
-	rm -f cache*.init
-	mysql $(DBOPTS) $$LSBDB <dbperms.sql
-	mysql $(DBOPTS) $(TMPLSBDB) <tmpdbperms.sql;
-
-dump_apps::
-	for table in $(APP_TABLES) ; \
-	do \
-	        set +e; \
-	        echo $$table; \
-	        mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | grep -v 'Server version' >$$table.sql;\
-		rm -f "$$PWD/$$table.init" && mysql $(DBOPTS) $$LSBDB -e "select * from $$table into outfile '$$PWD/$$table.init'";\
-	done
-
-# Optimization for spec part tables
-
-optimize::
-	for table in $(ELEMENTS) ; \
-	do \
-	        set +e; \
-	        echo $$table; \
-	        mysql $(DBOPTS) $$LSBDB -e "OPTIMIZE TABLE $$table"; \
-	done
+restore_apps: $(APP_TABLES_RESTORE) cache permissions
+
+dump_apps: $(APP_TABLES_DUMP)
+
+clean:
+	rm -f elements_list



More information about the lsb-messages mailing list