maria-developers team mailing list archive
-
maria-developers team
-
Mailing list archive
-
Message #02251
Rev 10: Do one thing, and do that one thing proper. - The Unix Way. in file:///Users/hakan/work/monty_program/mariadb-tools/
At file:///Users/hakan/work/monty_program/mariadb-tools/
------------------------------------------------------------
revno: 10
revision-id: hakan@xxxxxxxxxxxx-20100226162413-118168qx5c9c02ml
parent: hakan@xxxxxxxxxxxx-20100219052704-8nnlzsbu0nslnxba
committer: Hakan Kuecuekyilmaz <hakan@xxxxxxxxxxxx>
branch nick: mariadb-tools
timestamp: Fri 2010-02-26 16:24:13 +0000
message:
Do one thing, and do that one thing proper. - The Unix Way.
We are running the benchmark for one source repository now. Also we run each test three times, so
that peaks and glitches are balanced out.
=== modified file 'sysbench/run-sysbench.sh'
--- a/sysbench/run-sysbench.sh 2010-02-19 05:27:04 +0000
+++ b/sysbench/run-sysbench.sh 2010-02-26 16:24:13 +0000
@@ -2,8 +2,10 @@
#
# Run sysbench tests with MariaDB and MySQL
#
-# Note: Do not run this script with root privileges.
-# We use killall -9, which can cause severe side effects!
+# Notes:
+# * Do not run this script with root privileges. We use
+# killall -9, which can cause severe side effects!
+# * By bzr pull we mean bzr merge --pull
#
# Hakan Kuecuekyilmaz <hakan at askmonty dot org> 2010-02-19.
#
@@ -19,6 +21,29 @@
exit 1
fi
+if [ $# != 3 ]; then
+ echo '[ERROR]: Please provide exactly three options.'
+ echo " Example: $0 [pull | no-pull] [/path/to/bzr/repo] [name]"
+ echo " $0 pull ${HOME}/work/monty_program/maria-local-master MariaDB"
+
+ exit 1
+else
+ PULL="$1"
+ LOCAL_MASTER="$2"
+ PRODUCT="$3"
+fi
+
+#
+# Binaries.
+#
+MYSQLADMIN='client/mysqladmin'
+
+#
+# Adjust the following paths according to your installation.
+#
+SYSBENCH='/usr/local/bin/sysbench'
+BZR='/usr/local/bin/bzr'
+
#
# Variables.
#
@@ -27,32 +52,42 @@
MY_SOCKET="${TEMP_DIR}/mysql.sock"
MYSQLADMIN_OPTIONS="--no-defaults -uroot --socket=$MY_SOCKET"
MYSQL_OPTIONS="--no-defaults \
+ --datadir=$DATA_DIR \
+ --language=./sql/share/english \
+ --max_connections=256 \
+ --query_cache_size=0 \
+ --query_cache_type=0 \
--skip-grant-tables \
- --language=./sql/share/english \
- --datadir=$DATA_DIR \
- --tmpdir=$TEMP_DIR \
--socket=$MY_SOCKET \
--table_open_cache=512 \
--thread_cache=512 \
- --query_cache_size=0 \
- --query_cache_type=0 \
+ --tmpdir=$TEMP_DIR \
+ --innodb_additional_mem_pool_size=32M \
+ --innodb_buffer_pool_size=1024M \
+ --innodb_data_file_path=ibdata1:32M:autoextend \
--innodb_data_home_dir=$DATA_DIR \
- --innodb_data_file_path=ibdata1:128M:autoextend \
- --innodb_log_group_home_dir=$DATA_DIR \
- --innodb_buffer_pool_size=1024M \
- --innodb_additional_mem_pool_size=32M \
- --innodb_log_file_size=256M \
- --innodb_log_buffer_size=16M \
+ --innodb_doublewrite=0 \
--innodb_flush_log_at_trx_commit=1 \
+ --innodb_flush_method=O_DIRECT \
--innodb_lock_wait_timeout=50 \
- --innodb_doublewrite=0 \
- --innodb_flush_method=O_DIRECT \
- --innodb_thread_concurrency=0 \
- --innodb_max_dirty_pages_pct=80"
+ --innodb_log_buffer_size=16M \
+ --innodb_log_file_size=256M \
+ --innodb_log_group_home_dir=$DATA_DIR \
+ --innodb_max_dirty_pages_pct=80 \
+ --innodb_thread_concurrency=0"
+# Number of threads we run sysbench with.
NUM_THREADS="1 4 8 16 32 64 128"
+
+# The table size we use for sysbench.
TABLE_SIZE=2000000
+
+# The run time we use for sysbench.
RUN_TIME=300
+
+# How many times we run each test.
+LOOP_COUNT=3
+
SYSBENCH_TESTS="delete.lua \
insert.lua \
oltp_complex_ro.lua \
@@ -61,6 +96,7 @@
select.lua \
update_index.lua \
update_non_index.lua"
+
SYSBENCH_OPTIONS="--oltp-table-size=$TABLE_SIZE \
--max-time=$RUN_TIME \
--max-requests=0 \
@@ -68,123 +104,86 @@
--mysql-user=root \
--mysql-engine-trx=yes"
-PRODUCTS='MariaDB MySQL'
-
# Timeout in seconds for waiting for mysqld to start.
TIMEOUT=100
#
# Files
#
-MARIADB_BUILD_LOG='/tmp/mariadb_build.log'
-MYSQL_BUILD_LOG='/tmp/mysql_build.log'
+BUILD_LOG="/tmp/${PRODUCT}_build.log"
#
# Directories.
#
BASE="${HOME}/work"
-MARIADB_LOCAL_MASTER="${BASE}/monty_program/maria-local-master"
-MARIADB_WORK="${BASE}/monty_program/maria"
-MYSQL_LOCAL_MASTER="${BASE}/mysql/mysql-server-local-master"
-MYSQL_WORK="${BASE}/mysql/mysql-server"
TEST_DIR="${BASE}/monty_program/sysbench/sysbench/tests/db"
RESULT_DIR="${BASE}/sysbench-results"
-
-#
-# Binaries.
-#
-MYSQLADMIN='./client/mysqladmin'
-SYSBENCH='/usr/local/bin/sysbench'
-BZR='/usr/local/bin/bzr'
-
-#
-# Refresh repositories.
-#
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Refreshing source repositories."
-rm -rf $MARIADB_WORK
-if [ ! -d $MARIADB_LOCAL_MASTER ]; then
- echo "[ERROR]: Local master of MariaDB does not exist."
- echo " Please make a initial branch from lp:maria"
- echo " Exiting."
- exit 1
-else
- cd $MARIADB_LOCAL_MASTER
+WORK_DIR='/tmp'
+
+if [ ! -d $LOCAL_MASTER ]; then
+ echo "[ERROR]: Supplied local master $LOCAL_MASTER does not exists."
+ echo " Please provide a valid bzr repository."
+ echo " Exiting."
+ exit 1
+fi
+
+#
+# Refresh repositories, if requested.
+#
+if [ x"$PULL" = x"pull" ]; then
+ echo "[$(date "+%Y-%m-%d %H:%M:%S")] Refreshing source repositories."
+
+ cd $LOCAL_MASTER
echo "Pulling latest MariaDB sources."
- $BZR pull
- if [ $? != 0 ]; then
- echo "[ERROR]: $BZR pull for $MARIADB_LOCAL_MASTER failed"
- echo " Please check your bzr setup"
- echo " Exiting."
- exit 1
- fi
-
- echo "Branching MariaDB working directory."
- $BZR branch $MARIADB_LOCAL_MASTER $MARIADB_WORK
- if [ $? != 0 ]; then
- echo "[ERROR]: $BZR branch of $MARIADB_LOCAL_MASTER failed"
- echo " Please check your bzr setup"
- echo " Exiting."
- exit 1
- fi
-fi
-
-rm -rf $MYSQL_WORK
-if [ ! -d $MYSQL_LOCAL_MASTER ]; then
- echo "[ERROR]: Local master of MySQL does not exist."
- echo " Please make a initial branch from lp:mysql-server"
- echo " Exiting."
- exit 1
-else
- cd $MYSQL_LOCAL_MASTER
- echo "Pulling latest MySQL sources."
- $BZR pull
- if [ $? != 0 ]; then
- echo "[ERROR]: $BZR pull for $MYSQL_LOCAL_MASTER failed"
- echo " Please check your bzr setup"
- echo " Exiting."
- exit 1
- fi
-
- echo "Branching MySQL working directory."
- $BZR branch $MYSQL_LOCAL_MASTER $MYSQL_WORK
- if [ $? != 0 ]; then
- echo "[ERROR]: $BZR branch of $MYSQL_LOCAL_MASTER failed"
- echo " Please check your bzr setup"
- echo " Exiting."
- exit 1
- fi
-fi
-
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Done refreshing source repositories."
-
-
-#
-# TODO: Add platform detection and choose proper build script.
-#
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Starting to compile."
-
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Compiling MariaDB."
-cd $MARIADB_WORK
-BUILD/compile-amd64-max > $MARIADB_BUILD_LOG 2>&1
-if [ $? != 0 ]; then
- echo "[ERROR]: Build of $MARIADB_WORK failed"
- echo " Please check the log at $MARIDB_BUILD_LOG"
- echo " Exiting."
- exit 1
-fi
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Finnished compiling MariaDB."
-
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Compiling MySQL."
-cd $MYSQL_WORK
-BUILD/compile-amd64-max > $MYSQL_BUILD_LOG 2>&1
-if [ $? != 0 ]; then
- echo "[ERROR]: Build of $MYSQL_WORK failed"
- echo " Please check the log at $MYSQL_BUILD_LOG"
- echo " Exiting."
- exit 1
-fi
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Finnished compiling MySQL."
-echo "[$(date "+%Y-%m-%d %H:%M:%S")] Finnished compiling."
+ $BZR merge --pull
+ if [ $? != 0 ]; then
+ echo "[ERROR]: $BZR pull for $LOCAL_MASTER failed"
+ echo " Please check your bzr setup and/or repository"
+ echo " Exiting."
+ exit 1
+ fi
+
+ echo "[$(date "+%Y-%m-%d %H:%M:%S")] Done refreshing source repositories."
+fi
+
+cd $WORK_DIR
+TEMP_DIR=$(mktemp -d)
+if [ $? != 0 ]; then
+ echo "[ERROR]: mktemp in $WORK_DIR failed."
+ echo 'Exiting.'
+
+ exit 1
+fi
+
+#
+# bzr export refuses to export to an existing directory,
+# therefore we use an extra build/ directory.
+#
+echo "Exporting from $LOCAL_MASTER to ${TEMP_DIR}/build"
+$BZR export --format=dir ${TEMP_DIR}/build $LOCAL_MASTER
+if [ $? != 0 ]; then
+ echo '[ERROR]: bzr export failed.'
+ echo 'Exiting.'
+
+ exit 1
+fi
+
+#
+# Compile sources.
+# TODO: Add platform detection and choose proper build script accordingly.
+#
+echo "[$(date "+%Y-%m-%d %H:%M:%S")] Starting to compile $PRODUCT."
+
+cd ${TEMP_DIR}/build
+BUILD/compile-amd64-max > $BUILD_LOG 2>&1
+if [ $? != 0 ]; then
+ echo "[ERROR]: Build of $PRODUCT failed"
+ echo " Please check your log at $BUILD_LOG"
+ echo " Exiting."
+ exit 1
+fi
+
+echo "[$(date "+%Y-%m-%d %H:%M:%S")] Finnished compiling $PRODUCT."
#
# Go to work.
@@ -194,78 +193,79 @@
#
# Prepare results directory.
#
-if [ ! -d $RESULT_DIRS ]; then
- echo "[NOTE]: $RESULT_DIRS did not exist."
+if [ ! -d $RESULT_DIR ]; then
+ echo "[NOTE]: $RESULT_DIR did not exist."
echo " We are creating it for you!"
- mkdir $RESULT_DIRS
+ mkdir $RESULT_DIR
fi
TODAY=$(date +%Y-%m-%d)
mkdir ${RESULT_DIR}/${TODAY}
-
-for PRODUCT in $PRODUCTS; do
- mkdir ${RESULT_DIR}/${TODAY}/${PRODUCT}
-
- killall -9 mysqld
- rm -rf $DATA_DIR
- rm -f $MY_SOCKET
- mkdir $DATA_DIR
-
- if [ x"$PRODUCT" = x"MariaDB" ];then
- cd $MARIADB_WORK
- else
- cd $MYSQL_WORK
- fi
-
- sql/mysqld $MYSQL_OPTIONS &
-
- j=0
- STARTED=-1
- while [ $j -le $TIMEOUT ]
- do
- $MYSQLADMIN $MYSQLADMIN_OPTIONS ping > /dev/null 2>&1
- if [ $? = 0 ]; then
- STARTED=0
-
- break
- fi
-
- sleep 1
- j=$(($j + 1))
- done
-
- if [ $STARTED != 0 ]; then
- echo '[ERROR]: Start of mysqld failed.'
- echo ' Please check your error log.'
- echo ' Exiting.'
-
- exit 1
- fi
-
- for SYSBENCH_TEST in $SYSBENCH_TESTS; do
- mkdir ${RESULT_DIR}/${TODAY}/${PRODUCT}/${SYSBENCH_TEST}
-
- for THREADS in $NUM_THREADS; do
- THIS_RESULT_DIR="${RESULT_DIR}/${TODAY}/${PRODUCT}/${SYSBENCH_TEST}/${THREADS}"
- mkdir $THIS_RESULT_DIR
- echo "[$(date "+%Y-%m-%d %H:%M:%S")] Running $SYSBENCH_TEST with $THREADS threads for $PRODUCT"
-
- $MYSQLADMIN $MYSQLADMIN_OPTIONS -f drop sbtest
- $MYSQLADMIN $MYSQLADMIN_OPTIONS create sbtest
- if [ $? != 0 ]; then
- echo "[ERROR]: Create of sbtest database failed"
- echo " Please check your setup."
- echo " Exiting"
- exit 1
- fi
-
- SYSBENCH_OPTIONS="$SYSBENCH_OPTIONS --num-threads=$THREADS --test=${TEST_DIR}/${SYSBENCH_TEST}"
- $SYSBENCH $SYSBENCH_OPTIONS prepare
- $SYSBENCH $SYSBENCH_OPTIONS run > ${THIS_RESULT_DIR}/result.txt 2>&1
-
- done
- done
+mkdir ${RESULT_DIR}/${TODAY}/${PRODUCT}
+
+killall -9 mysqld
+rm -rf $DATA_DIR
+rm -f $MY_SOCKET
+mkdir $DATA_DIR
+
+sql/mysqld $MYSQL_OPTIONS &
+
+j=0
+STARTED=-1
+while [ $j -le $TIMEOUT ]
+ do
+ $MYSQLADMIN $MYSQLADMIN_OPTIONS ping > /dev/null 2>&1
+ if [ $? = 0 ]; then
+ STARTED=0
+
+ break
+ fi
+
+ sleep 1
+ j=$(($j + 1))
+done
+
+if [ $STARTED != 0 ]; then
+ echo '[ERROR]: Start of mysqld failed.'
+ echo ' Please check your error log.'
+ echo ' Exiting.'
+
+ exit 1
+fi
+
+for SYSBENCH_TEST in $SYSBENCH_TESTS
+ do
+ mkdir ${RESULT_DIR}/${TODAY}/${PRODUCT}/${SYSBENCH_TEST}
+
+ for THREADS in $NUM_THREADS
+ do
+ THIS_RESULT_DIR="${RESULT_DIR}/${TODAY}/${PRODUCT}/${SYSBENCH_TEST}/${THREADS}"
+ mkdir $THIS_RESULT_DIR
+ echo "[$(date "+%Y-%m-%d %H:%M:%S")] Running $SYSBENCH_TEST with $THREADS threads and $LOOP_COUNT iterations for $PRODUCT" | tee ${THIS_RESULT_DIR}/results.txt
+ echo '' >> ${THIS_RESULT_DIR}/results.txt
+
+ k=0
+ while [ $k -lt $LOOP_COUNT ]
+ do
+ $MYSQLADMIN $MYSQLADMIN_OPTIONS -f drop sbtest
+ $MYSQLADMIN $MYSQLADMIN_OPTIONS create sbtest
+ if [ $? != 0 ]; then
+ echo "[ERROR]: Create of sbtest database failed"
+ echo " Please check your setup."
+ echo " Exiting"
+ exit 1
+ fi
+
+ SYSBENCH_OPTIONS="$SYSBENCH_OPTIONS --num-threads=$THREADS --test=${TEST_DIR}/${SYSBENCH_TEST}"
+ $SYSBENCH $SYSBENCH_OPTIONS prepare
+ $SYSBENCH $SYSBENCH_OPTIONS run > ${THIS_RESULT_DIR}/result${k}.txt 2>&1
+
+ grep "write requests:" ${THIS_RESULT_DIR}/result${k}.txt | awk '{ print $4 }' | sed -e 's/(//' >> ${THIS_RESULT_DIR}/results.txt
+
+ k=$(($k + 1))
+ done
+ done
done
#
