分表分库方法总结

案例一:

1,背景:一个地址薄的应用程序,设计的用户量为2亿,统计出每个用户的地址薄为30个左右,整个数据量为60亿,使用mysql数据库

计划分为:1000个表,100个库

2,分库分表代码

?





1

2

3

4

5

6

7

8

private function getDbNo($email

    $m
= md5($email); 

    $n
= hexdec(substr($m, 0, 16)); 

    $tableNo
= fmod($n, 1000); 

    $dbNo
= $tableNo
% 100; 

    return
array($dbNo, $tableNo); 

  3,配合的连接访问代码

?





1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

require_once
‘Db/Config.php‘;

class Db_Adapter

{

    const
MASTER    = 0;

    const
SLAVE     = 1;

    private
static $instances = array();

    private
$conf = array();

    private
$conns = array();

    private
$conn = NULL;

    private
$stmt = NULL;

    

    public
function __construct($conf)

    {

        $this->conf = $conf;

    }

    public
function execute($sql, $params)

    {

        $cmd
= substr(strtolower(trim($sql)), 0, 6);

        if
($cmd == ‘select‘) {

            $conn
= $this->getConn(self::SLAVE);

        } else
{

            $conn
= $this->getConn(self::MASTER);

        }

        $conn->prepare($sql);

        $stmt
= $conn->execute($params);

        $this->conn = $conn;

        $this->stmt = $stmt;

    }

    public
function fetch()

    {

        return
$this->stmt->fetch();

    }

    public
function fetchAll()

    {

        return
$this->stmt->fetchAll();

    }

    public
function lastInsertId($name
= NULL)

    {

        return
$this->conn->lastInsertId($name);

    }

    public
function rowCount()

    {

        return
$this->stmt->rowCount();

    }

    private
function getConn($type)

    {

        if
($type == self::SLAVE && isset($this->conf[self::SLAVE])) {

            $id
= 0;

        } else
{

            $id
= 1;

        }

        if
(!isset($this->conns[$id])) {

            $conf
= $this->conf[$id];

            $this->conns[$id] = new
PDO(

                $conf[‘dsn‘], $conf[‘user‘], $conf[‘pass‘],

                self::dbOptions);

        }

        return
$this->conns[$id];

    }

    public
static function getInstance($dbName, $dbNo
= 0)

    {

        $key
= $dbName
. ‘_‘ . $dbNo;

        if
(!isset(self::$instances[$key])) {

            $conf
= Db_Config::getConfig($dbName, $dbNo); //连接配置参数

            self::$instances[$key] = new
self($conf);

        }

        return
self::$instances[$key];

    }

}

  
4,潜在问题

如果某个表中的那些用户的地址薄联系人超多,如每个人1000个,则可能出现该表超大,需要把该表区分为子表,暂时没有配置中心来处理该情况。

(若真的出现该情况,在连接参数这个地方继续作一次hash)。

转于:http://hanyh.iteye.com/blog/431323

案例二:

?





1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

<?php

include ‘config.php‘;

class Model{

        //用户名

        protected
$user;

        //密码

        protected
$pwd;

        //主机

        protected
$host;

        //库名,是一个数组

        protected
$dbName=array();

        //字符集

        protected
$charset=‘utf8‘;

        //连接资源是一个数组

        protected
$_link=array();

        //通用表名

        protected
$tabName;

        //真实表名

        protected
$trueTabName;

        //表前缀

        protected
$prefix;

        //字段缓存

        protected
$fields;

        //创建表的sql语句

        protected
$createSql=‘CREATE TABLE IF NOT EXISTS __TABLENAME__(

  `id` mediumint(9) NOT NULL AUTO_INCREMENT,

  `username` char(15) NOT NULL,

  `password` char(32) NOT NULL,

  `createtime` int(11) NOT NULL,

  PRIMARY KEY (`id`)

) ENGINE=MyISAM  DEFAULT CHARSET=utf8 AUTO_INCREMENT=1;‘;

//1,通过ID取余,得到真实表名    mod

//2,用户名截取前几位   substr

//3,md5                        md5

//4,不带分库分表        none

        protected
$partition=array(

                ‘type‘=>‘md5‘,       

                

                ‘rule‘=>1,

        

        );

        public
function __construct($tabName=‘‘){

                $this->user=DB_USER;

                $this->host=DB_HOST;

                $this->dbName[0]=DB_NAME;

                $this->charset=DB_CHARSET;

                $this->prefix=DB_PREFIX;

                $this->pwd=DB_PWD;

                if(empty($tabName)){

                //userModel

                //newModel

                        $this->tabName=$this->prefix.ucfirst(strtolower(substr(get_class($this),0,-5)));

                }else{

                        $this->tabName=$this->prefix.$tabName;

                }

                $this->_link[0]=$this->connect($this->host,$this->user,$this->pwd,$this->dbName,$this->charset);

        }

        public
function connect($host,$user,$pwd,$dbName,$charset,$linkId=0){

                $conn=mysql_connect($host,$user,$pwd);

                

                if(mysql_errno()){

                        $this->error(-1,$conn);

                        return
false;

                }

        

                if(!$this->selectDb($dbName[$linkId],$conn)){

                        $this->error(-2,$conn);

                        return
false;       

                }

                

                if(!$this->setCharset($charset,$conn)){

                        $this->error(-3,$conn);

                        return
false;

                }

        

                return
$conn;

        }

        public
function selectDb($dbName,$conn){

                if(mysql_select_db($dbName,$conn)){

                        return
true;

                }else{

                        return
false;

                }       

        }

        public
function setCharset($charset,$conn){

                if(mysql_set_charset($charset,$conn)){

                        return
true;

                }else{

                        return
false;

                }

        }

        public
function addServer($host,$user,$pwd,$dbName,$charset,$linkId){

                $this->dbName[$linkId]=$dbName;

                $this->_link[$linkId]=$this->connect($host,$user,$pwd,$dbName,$charset,$linkId);

        }

        public
function getTrueTable($content,$linkId=0){

                switch($this->partition[‘type‘]){

                        case
‘mod‘:

                                if(!is_int($content)){

                                        $this->error(-4);

                                        return
false;

                                }

                                $string=$content%$this->partition[‘rule‘];

                                break;

                        case
‘substr‘:

                                $string=substr($content,0,$this->partition[‘rule‘]);

                                break;

                        case
‘md5‘:

                                $string=substr(md5($content),0,$this->partition[‘rule‘]);

                                break;

                        case
‘none‘:

                                $string=null;

                                break;

                }

                if(empty($string)){

                        $this->trueTableName=$this->tabName;

                }else{

                        $this->trueTableName=$this->tabName.‘_‘.$string;

                }

                //第一,判断表是否存在,存在返回表字段缓存

                //第二,不存在,则创建表,返回字段缓存

                        $this->existsTable($this->trueTableName,$linkId);

        }

        //表是否存在

        //是否缓存了字段

        protected
function existsTable($tableName,$linkId=0){

                $database=$this->dbName[$linkId];

                $sql=‘select `TABLE_NAME` from `INFORMATION_SCHEMA`.`TABLES` where `TABLE_SCHEMA`=\‘‘.$database.‘\‘ and `TABLE_NAME`=\‘‘.$tableName.‘\‘‘;

        

                if($this->execute($sql,$linkId)){

                        //表存在

                        if(file_exists(‘cache/‘.md5($this->tabName).‘.php‘)){

                                $this->fields=include
‘cache/‘.md5($this->tabName).‘.php‘;

                        }else{

                                //暂时留着不写,待会来写

                                $this->fields=$this->getFieldCache($linkId);

                        }

                }else{

                        //表不存在

                        $this->createTable($this->trueTableName,$linkId);

                        $this->fields=$this->getFieldCache($linkId);

                }

        }

        protected
function getFieldCache($linkId=0){

                if(file_exists(‘cache/‘.md5($this->tabName).‘.php‘)){

                        $fields=include
‘cache/‘.md5($this->tabName).‘.php‘;

                        return
$fields;       

                }

                $sql="desc $this->trueTableName";

                $f=$this->query($sql,$linkId);

                

                $fields=$this->writeFields($f);

                return
$fields;

                

        }

        protected
function writeFields($f){

                foreach($f
as $key=>$value){

                        $fields[]=$value[‘Field‘];

                        

                        if($value[‘Key‘]==‘PRI‘){

                                $fields[‘_pk‘]=$value[‘Field‘];

                        }

                        if($value[‘Extra‘]==‘auto_increment‘){

                                $fields[‘_auto‘]=$value[‘Field‘];

                        }

                }

                $string="<?php \n return ".var_export($fields,true)."\n?>";

                

                file_put_contents(‘cache/‘.md5($this->tabName).‘.php‘,$string);

                return
$fields;

                        

        }

        protected
function createTable($tabName,$linkId=0){

                $sql=str_replace(‘__TABLENAME__‘,$tabName,$this->createSql);

        

                $this->execute($sql,$linkId);

        }

        //不需要返回结果集我用execute方法

        public
function  execute($sql,$linkId=0){

                $conn=$this->_link[$linkId];

        

                $result=mysql_query($sql,$this->_link[$linkId]);

                if($result&&mysql_affected_rows()){

                

                        return
mysql_affected_rows();

                }else{

                        return
false;

                }

        }

        //需要返回结果集我用query方法

        public
function query($sql,$linkId=0){

                $result=mysql_query($sql,$this->_link[$linkId]);

                

                if($result&&mysql_affected_rows()){

                        while($row=mysql_fetch_assoc($result)){

                                $rows[]=$row;

                        }

                }else{

                        return
false;

                }

                return
$rows;

        }

        public
function error($num,$conn){

                switch($num){

                        case
-1:

                                $string=‘连接数据库服务器失败‘.mysql_error($conn);

                                break;

                        case
-2:

                                $string=‘选择数据失败‘;

                                break;

                        case
-3:

                                $string=‘设置字符集失败‘;

                                break;

                        case
-4:

                                $string=‘数据库路由时选择的是取余,传入的不是整型‘;

                                break;

                }

        }

        

        //查最大值

        public
function max($field,$linkId=0){

                if(!in_array($field,$this->fields)){

                        return
false;

                }

                $sql="select max($field) as re from $this->trueTableName";

                $result=$this->query($sql,$linkId);

                $row=$result[‘re‘];

                return
$row;

                

        }       

        

        //查最小值

        public
function min($field,$linkId=0){

                if(!in_array($field,$this->fields)){

                        return
false;

                }

                $sql="select min($field) as re from $this->trueTableName";

                $result=$this->query($sql,$linkId);

                $row=$result[‘re‘];

                return
$row;

                

        }

        //求和

        public
function sum($field,$linkId=0){

                if(!in_array($field,$this->fields)){

                        return
false;

                }

                $sql="select sum($field) as re from $this->trueTableName";

                $result=$this->query($sql,$linkId);

                $row=$result[‘re‘];

                return
$row;

                

        }

        //最平均数

        public
function avg($field,$linkId=0){

                if(!in_array($field,$this->fields)){

                        return
false;

                }

                $sql="select avg($field) as re from $this->trueTableName";

                $result=$this->query($sql,$linkId);

                $row=$result[‘re‘];

                return
$row;

                

        }

        //求总数

        public
function count($field=‘‘,$linkId=0){

                if(empty($field)){

                        $field=$this->fields[‘_pk‘];

                }

                $sql="select count($field) as re from $this->trueTableName";

                $result=$this->query($sql,$linkId);

                $row=$result[‘re‘];

                return
$row;

        }

        //

        //删除

        public
function delete($data,$where=‘‘,$linkId=0,$order=‘‘,$limit=‘‘){

                //delete from 表  where 字段  order by  字段 limit

                

                if(is_array($data)){

                        $value=join(‘,‘,$data);

                }else{

                        $value=(int)$data;

                }

                $fields=$this->fields[‘_pk‘];

                if(empty($where)){

                        $sql="delete from $this->trueTableName where $fields in ($value)";

                }else{

                        $where=‘where ‘.$where;

                        if(!empty($order)){

                                $order=‘order by ‘.$order;       

                        }

                        if(!empty($limit)){

                                $limit=‘limit ‘.$limit;       

                        }

                        $sql="delete from $this->trueTableName $where $order $limit";

                }

                return
$this->execute($sql,$linkId);

        }

        //

        //修改

        public
function save($data,$where,$linkId=0,$order=‘‘,$limit=‘‘){

                //update 表  set 字段=值,字段=值 where 条件 order  limit

                $key=array_keys($data);

                $newKey=array_intersect($key,$this->fields);

        

                foreach($data
as $key=>$value){

                        if(!in_array($key,$newKey))

                                continue;

                        $update.=$key.‘="‘.$value.‘",‘;

                

                }

                $update=rtrim($update,‘,‘);

                

                if(!empty($order)){

                        $order=‘order by ‘.$order;

                }

                if(!empty($limit)){

                        $limit=‘limit ‘.$limit;

                }

                if(!empty($where)){

                        $where=‘where ‘.$where;

                }

                $sql="update $this->trueTableName set $update $where $order $limit";

                echo
$sql;

                $result=$this->execute($sql,$linkId);

                return
$result;

        }

        //增加

        public
function add($data,$linkId=0){

                //insert into 表(字段) values(值)

                $key=array_keys($data);

                $newKey=array_intersect($key,$this->fields);

                foreach($data
as $key=>$value){

                        if(!in_array($key,$newKey))

                                continue;

                        $values.="‘".$value."‘,";

                }

                $values=trim($values,‘,‘);

                $fields=join(‘,‘,$newKey);

                $sql="insert into $this->trueTableName($fields) values($values)";

                echo
$sql;

                $result=$this->execute($sql,$linkId);

                return
$result;

        }

        //单条查询

        public
function find($linkId=0,$where=‘‘,$order=‘‘){

                //select * from 表 where  order  limit 1

                $field=join(‘,‘,$this->fields);

                if(!empty($where)){

                        $where=‘where ‘.$where;

                }

                if(!empty($order)){

                        $order=‘order by ‘.$order;

                }

                $sql="select $field from $this->trueTableName $where $order limit 1";

                $result=$this->query($sql,$linkId);

                return
$result[0];

                

        }

        //多条查询

        public
function select($field=‘‘,$linkId=0,$where=‘‘,$order=‘‘,$limit=‘‘){

                //select * from 表 where  order  limit

                if(empty($field)){

                        $fields=join(‘,‘,$this->fields);

                }else{

                        if(is_array($field)){

                                $newKey=array_intersect($field,$this->fields);

                                $fields=implode(‘,‘,$newKey);

                        }else{

                                $fields=$field;

                        }

                }

                if(!empty($where)){

                        $where=‘where ‘.$where;

                }

                if(!empty($order)){

                        $order=‘order by ‘.$order;

                }

                if(!empty($limit)){

                        $limit=‘limit ‘.$limit;

                }

                $sql="select $fields from $this->trueTableName $where $order $limit";

                $result=$this->query($sql,$linkId);

                return
$result;

                

        }

        //按照字段来查询数据

        function
__call($name,$param){

                $key=substr($name,0,5);

                if(strtolower($key)==‘getby‘){

                        $field=strtolower(substr($name,5));

                

                        if(!in_array($field,$this->fields)){

                                return
false;

                        }

                        $f=join(‘,‘,$this->fields);

                        $value=$param[0];

                        $sql="select $f  from $this->trueTableName where $field=‘$value‘";

        

                        $result=$this->query($sql);

                        return
$result[0];

                }

        }

}

?>

  

分表分库方法总结

时间: 2024-10-10 08:43:51

分表分库方法总结的相关文章

关于论坛数据库的设计(分表分库等-转)

关于论坛数据库的设计 文章分类:数据库 一个简单的论坛系统 1:包含下列信息: 2:每天论坛访问量300万左右,更新帖子10万左右. 请给出数据库表结构设计,并结合范式简要说明设计思路. 一. 发帖主题和回复信息存放在一张表,并在这个表中增加user_name字段 对数据库的操作而言,检索数据的性能基本不会对数据造成很大的影响(精确查找的情况下),而对表与表之间的连接却会产生巨大的影响, 特别在有巨量数据的表之间:因此对问题的定位基本可以确定:在显示和检索数据时,尽量减少数据库的连接以及表与表之

Sharding-Jdbc实现分表分库

Sharding-Jdbc分表分库LogicTable数据分片的逻辑表,对于水平拆分的数据库(表),同一类表的总称.订单信息表拆分为2张表,分别是t_order_0.t_order_1,他们的逻辑表名为t_order.ActualTable在分片的数据库中真实存在的物理表.即上个示例中的t_order_0.t_order_1.DataNode数据分片的最小单元.由数据源名称和数据表组成,例:test_msg0.t_order_0.配置时默认各个分片数据库的表结构均相同,直接配置逻辑表和真实表对应

.NETCore 下支持分表分库、读写分离的通用 Repository

首先声明这篇文章不是标题党,我说的这个类库是 FreeSql.Repository,它作为扩展库现实了通用仓储层功能,接口规范参数 abp vnext,定义和实现基础的仓储层(CURD). 安装 dotnet add package FreeSql.Repository 定义 var fsql = new FreeSql.FreeSqlBuilder() .UseConnectionString(FreeSql.DataType.Sqlite, @"Data Source=|DataDirect

电商分表分库, 根据orderCode,uid方案思路

一.电商两种方案分库分表 1.根据订单号分表分库 分库方案: 根据订单号后3位, 取模分库 后3位规则: 1. 小于512 //图1中,我也不知道为什么运算完, 最大值不会超过512(捂脸哭) 2. 由Long userId, Long merchantId进制转换为3位数 //因为项目有商家和用户的维度, 一个商家对应对个用户 图1: 生成orderCode方法 根据配置区分哪张表分库 图2: xml配置分库的表 实现dataSource连接池, 判断分库的表 图3: xml配置分库的表 配置

数据库分表分库

产生原因 1.当数据达到了上亿级别单个库使用效率性能是十分低下的,当进行查询等操作时候,也是从根节点去找到子节点然后在找到叶节点 , 数亿条取一条数据 性能就不是很迅速,而且单机的存储量,链接数,并发量,处理能力十分有限. 这个时候数据库就容易遇到了系统瓶颈, 所以为了 降低性能,缩短查询时间,减少数据库的负担,会采取分表分库的方法 . 数据库分表分库思路:水平拆分和垂直拆分 垂直拆分:就是不同的表存储在不同的数据库,按照业务进行独立划分 ,主要通过'列'进行划分,将不常用的字段或者较大的字段拆

Mysql分表分库分析

对于大型的互联网应用,数据库单表的数据量可能达到千万甚至上亿级别,同时面临这高并发的压力.Master-Slave结构只能对数据库的读能力进行扩展,写操作还是集中在Master中,Master并不能无限制的挂接Slave库,如果需要对数据库的吞吐能力进行进一步的扩展,可以考虑采用分库分表的策略. 1.分表 在分表之前,首先要选中合适的分表策略(以哪个字典为分表字段,需要将数据分为多少张表),使数据能够均衡的分布在多张表中,并且不影响正常的查询.在企业级应用中,往往使用org_id(组织主键)做为

总结下Mysql分表分库的策略及应用

上月前面试某公司,对于mysql分表的思路,当时简要的说了下hash算法分表,以及discuz分表的思路,但是对于新增数据自增id存放的设计思想回答的不是很好(笔试+面试整个过程算是OK过了,因与个人预期的薪酬不太理想而忍痛放弃.),在此再深究下mysql 分表优化之类的设计思路方案.先来闲扯下发文目的: 为什么要分表和分区? 日常开发中我们经常会遇到大表的情况,所谓的大表是指存储了百万级乃至千万级条记录的表.这样的表过于庞大,导致数据库在查询和插入的时候耗时太长,性能低下,如果涉及联合查询的情

Mycat分表分库&amp;nbsp;

一.Mycat介绍 Mycat 是一个开源的分布式数据库系统,是一个实现了 MySQL 协议的的Server,前端用户可以把它看作是一个数据库代理,用 MySQL 客户端工具和命令行访问,而其后端可以用MySQL 原生(Native)协议与多个 MySQL 服务器通信,也可以用 JDBC 协议与大多数主流数据库服务器通信,其核心功能是分表分库,即将一个大表水平分割为 N 个小表,存储在后端 MySQL 服务器里或者其他数据库里. 二.Mycat基础环境搭建 首先需要下载Mycat必需的一些环境:

Mycat分表分库&amp;nbsp;&amp;nbsp;

一.Mycat介绍 Mycat 是一个开源的分布式数据库系统,是一个实现了 MySQL 协议的的Server,前端用户可以把它看作是一个数据库代理,用 MySQL 客户端工具和命令行访问,而其后端可以用MySQL 原生(Native)协议与多个 MySQL 服务器通信,也可以用 JDBC 协议与大多数主流数据库服务器通信,其核心功能是分表分库,即将一个大表水平分割为 N 个小表,存储在后端 MySQL 服务器里或者其他数据库里. 二.Mycat基础环境搭建 首先需要下载Mycat必需的一些环境: