road of mysql internals-build our own mysql

      前两天 参加了一个需求讨论会，有了一点想法，因此打算细细的咀嚼一下mysql的源码，看看你能不能从中受到启发。因此本文的形式有点流水账的形式，但是更多的是将自己的知识形成一个体系。这篇的内容相对来比较简单，但是涉及的东西也比较多，比较杂，对于那些想从源码出发的人可以有一个参考。
本文是在ubuntu平台上，使用mariadb的源码，通过CMAKE工具，编译源码，以及将自己从中遇到的问题进行了一下简单的总结。
需要注意的是本文是多实例mysql编译，即ubuntu自带了一个mysql，我在此基础之上，依据源码有编译了一个mysql实例。
platform :

ubuntu@:~/Desktop/sc$ cat /etc/lsb-release
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=12.04
DISTRIB_CODENAME=precise
DISTRIB_DESCRIPTION="Ubuntu 12.04.4 LTS"
ubuntu@:~/Desktop/sc$ uname -r
3.2.0-61-generic
ubuntu@:~/Desktop/sc$ uname -v
#93-Ubuntu SMP Fri May 2 21:33:33 UTC 2014

mysql source code download url :
wget
unzip&untar

tar -xvf mariadb-5.5.37.tar.gz

cmake & install 如何定制自己的特定目录，端口

ubuntu@:~/Desktop/sc/sc5.5.37/bld_debug$ cmake .. \
-DCMAKE_BUILD_TYPE=Debug  \
-DCMAKE_INSTALL_PREFIX=$HOME/ssql \
-DMYSQL_DATADIR=$HOME/ssql/var   \
-DMYSQL_UNIX_ADDR=$HOME/ssql/tmp/mysql.sock \
-DMYSQL_TCP_PORT=3307
-----
-- Configuring done
-- Generating done
-- Build files have been written to: /home/ubuntu/Desktop/sc/sc5.3.37/bld_debug

check option of config

ubuntu@:~/Desktop/sc/sc5.3.37/bld_debug$ cat  cmake.check | grep 'prefix'
// Install path prefix, prepended onto install directories.
ubuntu@:~/Desktop/sc/sc5.3.37/bld_debug$ cat  cmake.check | grep 'PREFIX'
CMAKE_INSTALL_PREFIX:PATH=/home/ubuntu/ssql
ubuntu@:~/Desktop/sc/sc5.3.37/bld_debug$ cat  cmake.check | grep 'DATADIR'
MYSQL_DATADIR:PATH=/home/ubuntu/ssql/var

测试，编译，安装

make test
make install

version info

ubuntu@:~/ssql/bin$ ./mysqld -V
./mysqld  Ver 5.5.37-MariaDB-debug for Linux on i686 (Source distribution)

第二实例的mysql授权表的初始化，

ubuntu@:~/ssql/scripts$ sudo ./mysql_install_db  --user=root --datadir=../var --basedir=..
[sudo] password for ubuntu:
Installing MariaDB/MySQL system tables in '../var' ...
140523  3:06:50 [Warning] Ignoring user change to 'root' because the user was set to 'mysql' earlier on the command line


Installation of system tables failed!  Examine the logs in
../var for more information.

The problem could be conflicting information in an external
my.cnf files. You can ignore these by doing:

    shell> ../scripts/scripts/mysql_install_db --defaults-file=~/.my.cnf

You can also try to start the mysqld daemon with:

    shell> ../bin/mysqld --skip-grant --general-log &

and use the command line tool ../bin/mysql
to connect to the mysql database and look at the grant tables:

    shell> ../bin/mysql -u root mysql
    mysql> show tables

Try 'mysqld --help' if you have problems with paths. Using
--general-log gives you a log in ../var that may be helpful.

针对DATADIR=VAR进行授权

root@ubuntu-virtual-machine:/home/ubuntu/ssql# cat /etc/group| grep 'mysql'
mysql:x:125:

root@ubuntu-virtual-machine:/home/ubuntu/ssql# cat  /etc/passwd | grep 'mysql'
mysql:x:114:125:MySQL Server,,,:/nonexistent:/bin/false

root@ubuntu-virtual-machine:/home/ubuntu/ssql# chown mysql var &&  ls -ali |grep 'var'
1059652 drwxr--r--  4 mysql  ubuntu   4096 May 23 03:06 var

ubuntu@:~/ssql/scripts$ sudo ./mysql_install_db --user=mysql --datadir=../var --basedir=.. --no-defaults
Warning: 1024 bytes lost, allocated at mysys/array.c:65, mysys/hash.c:92, mysys/thr_mutex.c:183, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104
Warning: 1024 bytes lost, allocated at mysys/array.c:65, mysys/hash.c:92, mysys/thr_mutex.c:178, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104
Warning: 112 bytes lost, allocated at mysys/mulalloc.c:51, mysys/thr_mutex.c:160, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104, extra/my_print_defaults.c:163
Memory lost: 2160 bytes in 3 chunks
Installing MariaDB/MySQL system tables in '../var' ...
Warning: 1 bytes lost, allocated at mysys/my_new.cc:33, sql/events.cc:826, sql/mysqld.cc:5020, sql/main.cc:26, 0x2d64d3, 0x8175731
Memory lost: 1 bytes in 1 chunks
OK
Filling help tables...
Warning: 1 bytes lost, allocated at mysys/my_new.cc:33, sql/events.cc:826, sql/mysqld.cc:5020, sql/main.cc:26, 0x4e34d3, 0x8175731
Memory lost: 1 bytes in 1 chunks
OK

usage of mysql_install-db:（权限表的特殊之处）
initial grant table && create var dir & & database mysql& test&& my.cnf
according to doc(url;http://dev.mysql.com/doc/refman/5.7/en/unix-postinstallation.html)

The script also creates privilege table entries for root and anonymous-user accounts. The accounts have no passwords initially.
Briefly, these privileges permit the MySQL root user to do anything, and permit
          anybody to create or use databases with a name of test or starting with test_.
For a more secure installation, invoke mysql_install_db with the --random-passwords option. This causes it to
          assign a random password to the MySQL root accounts, set the “password expired” flag for
          those accounts, and remove the anonymous-user MySQL accounts.

(verify this later by different mysql,root ,xxxx,and we will see different previllege)

ubuntu@:~/ssql/bin$ ./mysqld_safe --no-defaults --user=mysql &
./mysqld_safe: 138: ./mysqld_safe: cannot create /home/ubuntu/ssql/var/ubuntu-virtual-machine.err: Permission denied
rm: cannot remove `/home/ubuntu/ssql/var/ubuntu-virtual-machine.pid': Permission denied
./mysqld_safe: 1: eval: cannot create /home/ubuntu/ssql/var/ubuntu-virtual-machine.err: Permission denied
touch: cannot touch `/home/ubuntu/ssql/var/ubuntu-virtual-machine.err': Permission denied
chown: cannot access `/home/ubuntu/ssql/var/ubuntu-virtual-machine.err': Permission denied
chmod: cannot access `/home/ubuntu/ssql/var/ubuntu-virtual-machine.err': Permission denied
140523 03:41:43 mysqld_safe mysqld from pid file /home/ubuntu/ssql/var/ubuntu-virtual-machine.pid ended
./mysqld_safe: 138: ./mysqld_safe: cannot create /home/ubuntu/ssql/var/ubuntu-virtual-machine.err: Permission denied

ubuntu@:~/ssql$ sudo chmod -R a+rwx  var
ubuntu@:~/ssql$ cd var
ubuntu@:~/ssql/var$ ls
aria_log.00000001  aria_log_control  mysql  performance_schema test
ubuntu@:~/ssql/var$ ls -lai
total 40
1059652 drwxrwxrwx  5 mysql  ubuntu  4096 May 23 03:39 .
 942314 drwxrwxr-x 15 ubuntu ubuntu  4096 May 23 03:06 ..
1059102 -rwxrwxrwx  1 mysql  mysql  16384 May 23 03:39 aria_log.00000001
1059101 -rwxrwxrwx  1 mysql  mysql     52 May 23 03:39 aria_log_control
1059665 drwxrwxrwx  2 mysql  root    4096 May 23 03:39 mysql
1059175 drwxrwxrwx  2 mysql  mysql   4096 May 23 03:39 performance_schema
1059667 drwxrwxrwx  2 mysql  root    4096 May 23 03:06 test
ubuntu@:~/ssql/bin$ ./mysqld_safe --no-defaults
Warning: 1024 bytes lost, allocated at mysys/array.c:65, mysys/hash.c:92, mysys/thr_mutex.c:183, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104
Warning: 1024 bytes lost, allocated at mysys/array.c:65, mysys/hash.c:92, mysys/thr_mutex.c:178, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104
Warning: 112 bytes lost, allocated at mysys/mulalloc.c:51, mysys/thr_mutex.c:160, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104, extra/my_print_defaults.c:163
Memory lost: 2160 bytes in 3 chunks
Warning: 1024 bytes lost, allocated at mysys/array.c:65, mysys/hash.c:92, mysys/thr_mutex.c:183, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104
Warning: 1024 bytes lost, allocated at mysys/array.c:65, mysys/hash.c:92, mysys/thr_mutex.c:178, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104
Warning: 112 bytes lost, allocated at mysys/mulalloc.c:51, mysys/thr_mutex.c:160, mysys/thr_mutex.c:320, psi/mysql_thread.h:618, mysys/my_thr_init.c:339, mysys/my_thr_init.c:197, mysys/my_init.c:104, extra/my_print_defaults.c:163
Memory lost: 2160 bytes in 3 chunks
140523 03:44:41 mysqld_safe Logging to '/home/ubuntu/ssql/var/ubuntu-virtual-machine.err'.
140523 03:44:41 mysqld_safe Starting mysqld daemon with databases from /home/ubuntu/ssql/var

ubuntu@:~/ssql/bin$ ./mysql -uroot -p               [log in to the binary distribution]
Enter password:
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MySQL connection id is 55
Server version: 5.5.37-0ubuntu0.12.04.1 (Ubuntu)

Copyright (c) 2000, 2014, Oracle, Monty Program Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

   here i make a mistake,because of that there a two mysqld instance on my test pc.you will log in to the binary distribution if you
not specifed the port number.when you run script/mysql_install_db,it initialized the database and make mysqld run as the user you
specifed ,here is mysql,so the mysql user and mysql group must have the previlige to read and write the data dir
first instance

ubuntu@:~$ ps -e | grep 'mysql'
  946 ? 00:00:01 mysqld
ubuntu@:~$ mysql -uroot -p
Enter password:
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 40
Server version: 5.5.37-0ubuntu0.12.04.1 (Ubuntu)

Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> exit
Bye

source distribution

ubuntu@:~/ssql/bin$ ./mysqld_safe --no-defaults --user=mysql &
[1] 4089

mysqld_safe is the recommended way to start a mysqld server on Unix. mysqld_safe adds some safety features such as restarting the server when an error occurs and logging runtime information to an error log file. A description of error logging is given later in this section.

mysqld_safe tries to start an executable named mysqld. To override the default behavior and specify explicitly the name of the server you want to run, specify a --mysqld or --mysqld-version option to mysqld_safe.

ubuntu@:~$ ps -e | grep 'mysql'
  946 ? 00:00:01 mysqld
 4089 pts/0    00:00:00 mysqld_safe
 4168 pts/0    00:00:08 mysqld

ubuntu@:~/ssql/bin$ ./mysql --no-defaults --port=3307  -umysql -p
Enter password:
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 3
Server version: 5.5.37-MariaDB-debug Source distribution

Copyright (c) 2000, 2014, Oracle, Monty Program Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| test |
+--------------------+
2 rows in set (0.01 sec)

使用root以及匿名用户的测试，可以看到不用的用户看到的DB不同，初始密码均为空

ubuntu@:~/ssql/bin$ ./mysql --no-defaults --port=3307  -uroot -p
Enter password:
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 4
Server version: 5.5.37-MariaDB-debug Source distribution

Copyright (c) 2000, 2014, Oracle, Monty Program Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| mysql |
| performance_schema |
| test |
+--------------------+
4 rows in set (0.00 sec)

ubuntu@:~/ssql/bin$ ./mysql --no-defaults --port=3307  -uxxxx -p
Enter password:
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 5
Server version: 5.5.37-MariaDB-debug Source distribution

Copyright (c) 2000, 2014, Oracle, Monty Program Ab and others.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

MariaDB [(none)]> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| test |
+--------------------+
2 rows in set (0.00 sec)

check runnning after install

ubuntu@:~/ssql/bin$ ./mysqladmin --no-defaults version
./mysqladmin  Ver 9.0 Distrib 5.5.37-MariaDB, for Linux on i686
Copyright (c) 2000, 2014, Oracle, Monty Program Ab and others.

Server version        5.5.37-MariaDB-debug
Protocol version    10
Connection        Localhost via UNIX socket
UNIX socket        /home/ubuntu/ssql/tmp/mysql.sock
Uptime:            58 min 2 sec

Threads: 1  Questions: 48  Slow queries: 0  Opens: 15  Flush tables: 2  Open tables: 41  Queries per second avg: 0.013
ubuntu@:~/ssql/bin$ ./mysqlshow --no-defaults
+--------------------+
| Databases |
+--------------------+
| information_schema |
| test |
+--------------------+
ubuntu@:~/ssql/bin$ ./mysqladmin --no-defaults -uroot shutdown
[1]+ Done ./mysqld_safe --no-defaults --user=mysql

workbench test:

3306--> 3307

ubuntu@:~/ssql/mysql-test$ ./mysql-test-run --extern socket=../tmp/mysql.sock
Logging: ./mysql-test-run --extern socket=../tmp/mysql.sock
vardir: /home/ubuntu/ssql/mysql-test/var
Checking supported features...

实例数据库的安装
mysql mysql < file_name
employee data (large dataset, includes data and test/verification suite) Launchpad View US Ltr | A4
world database (MyISAM version, used in MySQL certifications and training) Gzip | Zip View US Ltr | A4
world database (InnoDB      version, used in MySQL certifications and training) Gzip | Zip View US Ltr | A4
sakila database (requires MySQL 5.0 or later) TGZ | Zip View US Ltr | A4
menagerie database
  下载地址 http://dev.mysql.com/doc/sakila/en/
1 download 
2 untar 
3 install
4  import to mysql 

ubuntu@:~/Downloads$ mysql -uroot -p  -D wmyisam < world.sql
Enter password:
ubuntu@:~/Downloads$ mysql -uroot -p  -D winnodb < world_innodb.sql
Enter password:

mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| mysql |
| sakila |
+--------------------+
3 rows in set (0.12 sec)
mysql> \u sakila;
ERROR 1049 (42000): Unknown database 'sakila;'
mysql> \u sakila
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> show tables;
+----------------------------+
| Tables_in_sakila |
+----------------------------+
| actor |
| actor_info |
| address |
| category |
| city |

mysql> show variables like 'data%' \G;
    *************************** 1. row ***************************
    Variable_name: datadir
            Value: /var/lib/mysql/
    1 row in set (0.00 sec)
    ERROR:
    No query specified
    mysql> system ls /var/lib/mysql/sakila
    actor.frm         del_film.TRN     nicer_but_slower_film_list.frm
    actor.ibd         film_actor.frm payment_date.TRN
    actor_info.frm         film_actor.ibd payment.frm
    address.frm         film_category.frm payment.ibd
    address.ibd         film_category.ibd payment.TRG
    category.frm         film.frm     rental_date.TRN
    category.ibd         film.ibd     rental.frm
    city.frm         film_list.frm rental.ibd
    city.ibd         film_text.frm rental.TRG
    country.frm         film_text.MYD sales_by_film_category.frm
    country.ibd         film_text.MYI sales_by_store.frm
    customer_create_date.TRN film.TRG     staff.frm

下面我们就开始源码的探索之路吧
source code client

#include "client_priv.h"
#include <m_ctype.h>
#include <stdarg.h>
#include <my_dir.h>
#ifndef __GNU_LIBRARY__
#define __GNU_LIBRARY__         // Skip warnings in getopt.h
#endif
#include "my_readline.h"
#include <signal.h>
#include <violite.h>

/* Common defines for all clients */

#include <my_global.h>
#include <my_sys.h>
#include <m_string.h>
#include <mysql.h>
#include <errmsg.h>
#include <my_getopt.h>

    connection的内部表示：

typedef struct st_mysql
{
  NET        net;            /* Communication parameters */
  unsigned char    *connector_fd;        /* ConnectorFd for SSL */
  char        *host,*user,*passwd,*unix_socket,*server_version,*host_info;
  char *info, *db;
  const struct charset_info_st *charset;
  MYSQL_FIELD    *fields;
  MEM_ROOT    field_alloc;
  my_ulonglong affected_rows;
  my_ulonglong insert_id;        /* id if insert on table with NEXTNR */
  my_ulonglong extra_info;        /* Not used */
  unsigned long thread_id;        /* Id for connection in server */
  unsigned long packet_length;
  unsigned int    port;
  unsigned long client_flag,server_capabilities;
  unsigned int    protocol_version;
  unsigned int    field_count;
  unsigned int     server_status;
  unsigned int server_language;
  unsigned int    warning_count;
  struct st_mysql_options options;
  enum mysql_status status;
  my_bool    free_me;        /* If free in mysql_close */
  my_bool    reconnect;        /* set to 1 if automatic reconnect */

  /* session-wide random string */
  char            scramble[SCRAMBLE_LENGTH+1];
  my_bool unused1;
  void *unused2, *unused3, *unused4, *unused5;

  LIST *stmts; /* list of all statements */
  const struct st_mysql_methods *methods;
  void *thd;
  /*
    Points to boolean flag in MYSQL_RES or MYSQL_STMT. We set this flag
    from mysql_stmt_close if close had to cancel result set of this object.
  */
  my_bool *unbuffered_fetch_owner;
  /* needed for embedded server - no net buffer to store the 'info' */
  char *info_buffer;
  void *extension;
} MYSQL;