[Lsb-messages] /var/www/bzr/lsb/devel/specdb r1254: Refactor specdb build/load/dump system.
Jeff Licquia
licquia at linuxfoundation.org
Tue Jan 22 02:00:26 UTC 2013
------------------------------------------------------------
revno: 1254
committer: Jeff Licquia <licquia at linuxfoundation.org>
branch nick: specdb
timestamp: Mon 2013-01-21 21:00:26 -0500
message:
Refactor specdb build/load/dump system.
This was prompted by continual dump/restore issues being seen by Mats
and me. New features/benefits:
- Individual tables can be dumped or restored.
- Procedure for dumping/restoring is no longer repeated in multiple
places; when changes need to be made, they can be made once.
- Support for handling the strange behavior of LOAD DATA INFILE in
MySQL. Since LOAD DATA INFILE with --local-infile is treated
differently than LOAD DATA LOCAL INFILE, we now support doing the
latter via the LOCAL_INFILE_CMD makefile setting.
- Related to the above, use DO_COPY to control whether to try and
dump/restore in-place, or use a temporary location as an
intermediate.
- Use the table lists in the makefile, rather than looping over *.sql
for lists of tables.
- Get rid of embedded shell loops generally in favor of proper make
dependencies.
- Break out most tasks into separate make targets. New targets:
dbsetup, permissions, alias_detector, optimize_speconly, clean.
- Break out permissions setting from the rest of the procedures, so
they are not run by default. This may mean that 'make permissions'
may need to be run manually when setting up a new database, but
prevents errors that occur when the DB user doesn't have GRANT
rights (as is currently the case with the default security
settings).
- As a supporting set of rules, create a list of elements for use by
the other rules, and clean it up with 'make clean'.
added:
.bzrignore
modified:
create_cache_tables_inits.sh
makefile
-------------- next part --------------
=== added file '.bzrignore'
--- a/.bzrignore 1970-01-01 00:00:00 +0000
+++ b/.bzrignore 2013-01-22 02:00:26 +0000
@@ -0,0 +1,1 @@
+elements_list
=== modified file 'create_cache_tables_inits.sh'
--- a/create_cache_tables_inits.sh 2011-12-12 07:35:05 +0000
+++ b/create_cache_tables_inits.sh 2013-01-22 02:00:26 +0000
@@ -13,9 +13,9 @@
LC_ALL=C cut cache_RLibRIntMapping.init -f2 | LC_ALL=C uniq >cache_RIntNames.init
LC_ALL=C cut RawInterface.init -f2,3,5 | LC_ALL=C sort -u --ignore-case | LC_ALL=C nl -w1 >cache_RIntCaseInsensitiveNames.init
-mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA INFILE '$PWD/cache_RIntNames.init' into table cache_RIntNames fields enclosed by '\''; optimize table cache_RIntNames"
-mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA INFILE '$PWD/cache_RLibRIntMapping.init' into table cache_RLibRIntMapping fields enclosed by '\''; optimize table cache_RLibRIntMapping"
-mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA INFILE '$PWD/cache_RIntCaseInsensitiveNames.init' into table cache_RIntCaseInsensitiveNames fields enclosed by '\''; optimize table cache_RIntCaseInsensitiveNames"
+mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA $LOCALCMD INFILE '$PWD/cache_RIntNames.init' into table cache_RIntNames fields enclosed by '\''; optimize table cache_RIntNames"
+mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA $LOCALCMD INFILE '$PWD/cache_RLibRIntMapping.init' into table cache_RLibRIntMapping fields enclosed by '\''; optimize table cache_RLibRIntMapping"
+mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "LOAD DATA $LOCALCMD INFILE '$PWD/cache_RIntCaseInsensitiveNames.init' into table cache_RIntCaseInsensitiveNames fields enclosed by '\''; optimize table cache_RIntCaseInsensitiveNames"
mysql -h $LSBDBHOST -u $LSBUSER --password=$LSBDBPASSWD $FILEOPTS $LSBDB -e "UPDATE cache_RIntCaseInsensitiveNames SET RIunmangled=NULL WHERE RIunmangled='' "
# For each LSB version, create a table with interfaces included in it (at least at one architecture)
=== modified file 'makefile'
--- a/makefile 2012-11-01 02:02:50 +0000
+++ b/makefile 2013-01-22 02:00:26 +0000
@@ -7,10 +7,16 @@
DBOPTS=-h $$LSBDBHOST -u $$LSBUSER --password=$$LSBDBPASSWD
DUMPOPTS=--quote-names --extended-insert=false --triggers=FALSE --skip-dump-date --skip-comments
-# Add '--local-infile' here if there are any problems with direct access to files
-FILEOPTS=
-# If TMPLSBDB environment variable is set, it will be used for temp database name;
-# otherwise we'll use "${LSBDB}_tmp"
+# Sometimes LOAD DATA INFILE doesn't work, but ...LOCAL INFILE does.
+# If this is your case, set this to 'local'.
+LOCAL_INFILE_CMD=
+# If DO_COPY is 'yes', then we copy files being loaded with LOAD DATA INFILE
+# to a common temporary location first.
+DO_COPY=no
+# Use this temporary location if needed.
+TMPDIR=/tmp
+# If TMPLSBDB environment variable is set, it will be used for temp database
+# name; otherwise we'll use "${LSBDB}_tmp"
TMPLSBDB=$${TMPLSBDB=${LSBDB}}_tmp
ELEMENTS=AbiApi AbiMacro ArchClass ArchConst ArchDE ArchES ArchInt \
@@ -29,42 +35,63 @@
JavaInterface RILMBuiltin RLibDeps RLibLink RLibRClass RLibRInt RawClass RawCommand RawILModule \
RawInterface RawLibSoname RawLibrary WeakSymbol
+ifeq (yes,$(DO_COPY))
+LOAD_LOCATION=$(TMPDIR)
+else
+LOAD_LOCATION=$$PWD
+endif
+
+# These variables hold the targets for each thing that needs to be done
+# in the form 'tablename_job'. Example: 'Type_dump' would dump the Type
+# table. Most of the work this makefile does is based on these rules.
+
+ELEMENTS_DUMP = $(shell echo $(ELEMENTS) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_dump"; done))
+COMMUNITY_DUMP = $(shell echo $(COMMUNITY) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_dump"; done))
+APP_TABLES_DUMP = $(shell echo $(APP_TABLES) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_dump"; done))
+ELEMENTS_RESTORE = $(shell echo $(ELEMENTS) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_restore"; done))
+COMMUNITY_RESTORE = $(shell echo $(COMMUNITY) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_restore"; done))
+APP_TABLES_RESTORE = $(shell echo $(APP_TABLES) | (while read line; do for word in $$line; do echo $$word; done; done) | (while read line; do echo "$${line}_restore"; done))
+
all:
@echo "Please specify dump or restore (or variants dumpall, restoreall, dump_apps, restore_apps)"
+# This is used to distinguish which dump and restore commands to use
+# for the data of a particular table.
+elements_list:
+ echo $(ELEMENTS) > $@
+
+%_dump: elements_list
+ mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $* | grep -v 'Server version' |grep -v 'character_set_client' > $*.sql
+ if grep -q $* elements_list; then \
+ mysqldump $(DBOPTS) $(DUMPOPTS) $$LSBDB $* | LC_ALL=C grep INSERT > $*.init; \
+ else \
+ rm -f "$$PWD/$*.init" && mysql $(DBOPTS) $$LSBDB -e "select * from $* into outfile '$$PWD/$*.init'"; \
+ fi
+
+%_restore: elements_list
+ @echo $*
+ @mysql $(DBOPTS) $$LSBDB < $*.sql
+ @if grep -q $* elements_list; then \
+ mysql $(DBOPTS) $$LSBDB < $*.init; \
+ else \
+ cp $*.init $(TMPDIR); \
+ chmod 644 $(TMPDIR)/$*.init; \
+ mysql $(DBOPTS) $$LSBDB -e "load data $(LOCAL_INFILE_CMD) infile '$(LOAD_LOCATION)/$*.init' into table $*"; \
+ rm -f $(TMPDIR)/$*.init; \
+ fi
+
# dump the "source code" tables
-dump::
- for table in $(ELEMENTS) ; \
- do \
- set +e; \
- echo $$table; \
- mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | grep -v 'Server version' |grep -v 'character_set_client' >$$table.sql;\
- mysqldump $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | LC_ALL=C grep INSERT >$$table.init;\
- done
+dump: $(ELEMENTS_DUMP)
# dump everything: the "source code" tables + the "community" tables
# if the community tables are populated, this will be big!
-dumpall::
- for table in `mysql $(DBOPTS) -e "SHOW TABLES" $$LSBDB | grep -v Tables | grep -v cache_` ;\
- do \
- set +e; \
- echo $$table; \
- mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | grep -v 'Server version' |grep -v 'character_set_client' >$$table.sql;\
- case "$(ELEMENTS)" in \
- *"$$table"*) mysqldump $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | LC_ALL=C grep INSERT >$$table.init ;; \
- *) rm -f "$$PWD/$$table.init" && mysql $(DBOPTS) $$LSBDB -e "select * from $$table into outfile '$$PWD/$$table.init'" ;; \
- esac;\
- done
+dumpall: dump $(COMMUNITY_DUMP)
-restore::
- for table in $(ELEMENTS) ; \
- do \
- set +e; \
- echo $$table; \
- mysql $(DBOPTS) $$LSBDB <$$table.sql; \
- mysql $(DBOPTS) $$LSBDB <$$table.init; \
- mysql $(DBOPTS) $$LSBDB -e "OPTIMIZE TABLE $$table"; \
- done
+dbsetup::
+ mysql $(DBOPTS) -e "drop database if exists $$LSBDB"
+ mysql $(DBOPTS) -e "drop database if exists $(TMPLSBDB)"
+ @mysqladmin $(DBOPTS) create $$LSBDB
+ @mysqladmin $(DBOPTS) create $(TMPLSBDB)
community_check::
@ret_code=0; \
@@ -82,34 +109,25 @@
fi; \
exit $$ret_code
-restoreall:: community_check
- mysql $(DBOPTS) -e "drop database if exists $$LSBDB"
- mysql $(DBOPTS) -e "drop database if exists $(TMPLSBDB)"
- @mysqladmin $(DBOPTS) create $$LSBDB
- @mysqladmin $(DBOPTS) create $(TMPLSBDB)
- #mysql $(DBOPTS) $$LSBDB <setupdb.sql;
-# sleep 5
- LC_ALL=C $(SHELL) -c 'for table in [A-Z]*sql ; \
- do \
- set +e; \
- table=`basename $$table .sql`; \
- echo $$table; \
- mysql $(DBOPTS) $$LSBDB <$$table.sql; \
- case "$(ELEMENTS)" in \
- *"$$table"*) mysql $(DBOPTS) $$LSBDB <$$table.init ;; \
- *) cp $${table}.init /tmp; chmod 644 /tmp/$${table}.init; mysql $(DBOPTS) $(FILEOPTS) $$LSBDB -e "load data infile \"/tmp/$${table}.init\" into table $$table"; rm /tmp/$${table}.init ;; \
- esac;\
- done'
- FILEOPTS=$(FILEOPTS) ./create_cache_tables_inits.sh
- mysql $(DBOPTS) $$LSBDB <create_cache_tables.sql;
+alias_detector::
mysql $(DBOPTS) $$LSBDB <create_alias_detector.sql;
- mysql $(DBOPTS) $$LSBDB <create_stored_procs.sql
- rm -f cache*.init
+
+optimize::
LC_ALL=C $(SHELL) -c 'for table in [A-Z]*sql ; \
do \
- table=`basename $$table .sql`; \
- mysql $(DBOPTS) $$LSBDB -e "SET SESSION myisam_sort_buffer_size = 30 * 1024 * 1024; OPTIMIZE TABLE $$table"; \
+ table=`basename $$table .sql`; \
+ mysql $(DBOPTS) $$LSBDB -e "SET SESSION myisam_sort_buffer_size = 30 * 1024 * 1024; OPTIMIZE TABLE $$table"; \
done'
+
+optimize_speconly::
+ for table in $(ELEMENTS) ; \
+ do \
+ set +e; \
+ echo $$table; \
+ mysql $(DBOPTS) $$LSBDB -e "OPTIMIZE TABLE $$table"; \
+ done
+
+permissions::
mysql $(DBOPTS) $$LSBDB <dbperms.sql;
mysql $(DBOPTS) $(TMPLSBDB) <tmpdbperms.sql;
@@ -117,54 +135,21 @@
# then call the 'cache' rule
cache::
-# FILEOPTS=$(FILEOPTS) ./create_cache_tables_inits.sh
+ LOCALCMD=$(LOCAL_INFILE_CMD) ./create_cache_tables_inits.sh
mysql $(DBOPTS) $$LSBDB <create_cache_tables.sql;
mysql $(DBOPTS) $$LSBDB <create_stored_procs.sql
rm -f cache*.init
- LC_ALL=C $(SHELL) -c 'for table in [A-Z]*sql ; \
- do \
- table=`basename $$table .sql`; \
- mysql $(DBOPTS) $$LSBDB -e "SET SESSION myisam_sort_buffer_size = 30 * 1024 * 1024; OPTIMIZE TABLE $$table"; \
- done'
- mysql $(DBOPTS) $$LSBDB <dbperms.sql;
- mysql $(DBOPTS) $(TMPLSBDB) <tmpdbperms.sql;
-
+
+restore: $(ELEMENTS_RESTORE) optimize_speconly
+
+restoreall:: community_check dbsetup $(ELEMENTS_RESTORE) $(COMMUNITY_RESTORE) \
+ alias_detector cache optimize
# rules to process application data only
-restore_apps::
- for table in $(APP_TABLES) ; \
- do \
- set +e; \
- echo $$table; \
- mysql $(DBOPTS) $$LSBDB <$$table.sql; \
- cp $$table.init /tmp; \
- chmod 644 /tmp/$$table.init; \
- mysql $(DBOPTS) $(FILEOPTS) $$LSBDB -e "load data infile '$$PWD/$$table.init' into table $$table";\
- rm /tmp/$$table.init; \
- done
- FILEOPTS=$(FILEOPTS) ./create_cache_tables_inits.sh
- mysql $(DBOPTS) $$LSBDB <create_cache_tables.sql;
- mysql $(DBOPTS) $$LSBDB <create_stored_procs.sql
- rm -f cache*.init
- mysql $(DBOPTS) $$LSBDB <dbperms.sql
- mysql $(DBOPTS) $(TMPLSBDB) <tmpdbperms.sql;
-
-dump_apps::
- for table in $(APP_TABLES) ; \
- do \
- set +e; \
- echo $$table; \
- mysqldump --add-drop-table --no-data $(DBOPTS) $(DUMPOPTS) $$LSBDB $$table | grep -v 'Server version' >$$table.sql;\
- rm -f "$$PWD/$$table.init" && mysql $(DBOPTS) $$LSBDB -e "select * from $$table into outfile '$$PWD/$$table.init'";\
- done
-
-# Optimization for spec part tables
-
-optimize::
- for table in $(ELEMENTS) ; \
- do \
- set +e; \
- echo $$table; \
- mysql $(DBOPTS) $$LSBDB -e "OPTIMIZE TABLE $$table"; \
- done
+restore_apps: $(APP_TABLES_RESTORE) cache permissions
+
+dump_apps: $(APP_TABLES_DUMP)
+
+clean:
+ rm -f elements_list
More information about the lsb-messages
mailing list