`
ahua186186
  • 浏览: 554281 次
  • 性别: Icon_minigender_1
  • 来自: 深圳
社区版块
存档分类
最新评论

分库分表思路

 
阅读更多

总体思路和切入点:

1.在spring数据访问封装层通过动态代理无侵入的扩展代码加入分库分表策略。


(1)分库:通过动态代理扩展SqlSession的代码并传入分库参数来选择sqlSessionTemplate的数据源的方式实现分库策略

public abstract class SqlSessionDaoSupport implements InitializingBean {
	private SqlSessionFactoryBean sqlSessionFactoryBean;

	private Map<DataSource, SqlSessionTemplate> dataSourceMap;

	private SqlSession sqlSession;

	{
		//通过动态代理扩展SqlSession的代码并传入分库参数来选择sqlSessionTemplate的数据源的方式实现分库策略
		sqlSession = (SqlSession) Proxy.newProxyInstance(SqlSessionDaoSupport.class.getClassLoader(),
				new Class[] { SqlSession.class }, new SessionHandler());
	}

	@Autowired(required = false)
	public final void setSqlSessionFactory(SqlSessionFactoryBean sqlSessionFactoryBean) {
		this.sqlSessionFactoryBean = sqlSessionFactoryBean;
	}

	public final SqlSession getSqlSession() {
		return sqlSession;
	}

	@Override
	public final void afterPropertiesSet() throws Exception {
		sqlSessionFactoryBean.afterPropertiesSet();
		//
		dataSourceMap = new LinkedHashMap<DataSource, SqlSessionTemplate>();
		dataSourceMap.put(sqlSessionFactoryBean.getMainDataSource(),
				new SqlSessionTemplate(sqlSessionFactoryBean.getMainSqlSessionFactory()));

		Map<String, DataSource> shardDataSources = sqlSessionFactoryBean.getShardDataSources();
		if (shardDataSources != null) {
			for (Entry<String, DataSource> entry : shardDataSources.entrySet()) {
				SqlSessionFactory sqlSessionFactory = sqlSessionFactoryBean.getShardSqlSessionFactory().get(
						entry.getKey());
				dataSourceMap.put(entry.getValue(), new SqlSessionTemplate(sqlSessionFactory));
			}
		}
	}

	private class SessionHandler implements InvocationHandler {

		@Override
		public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
			try {
				// 默认DataSource为MainDataSource
				DataSource targetDS = sqlSessionFactoryBean.getMainDataSource();

				//
				if (args == null || args.length == 0) {
					// 准备事务
					prepareTx(targetDS);
					//
					return method.invoke(dataSourceMap.get(sqlSessionFactoryBean.getMainDataSource()), args);
				}
				if (!(args[0] instanceof String)) {
					// 准备事务
					prepareTx(targetDS);
					//
					return method.invoke(dataSourceMap.get(sqlSessionFactoryBean.getMainDataSource()), args);
				}
				ShardParam shardParam = (args.length > 1 && args[1] instanceof ShardParam) ? (ShardParam) args[1]
						: null;
				if (shardParam == null) {
					// 准备事务
					prepareTx(targetDS);
					//
					return method.invoke(dataSourceMap.get(sqlSessionFactoryBean.getMainDataSource()), args);
				} else {
					args[1] = shardParam.getParams();
				}
				
				//
				String statement;
				String shardStrategyName;
				ShardStrategy shardStrategy;

				statement = (String) args[0];
				shardStrategyName = shardParam.getName();

				shardStrategy = sqlSessionFactoryBean.getShardStrategyMap().get(shardStrategyName);
				if (shardStrategy == null) {
					shardStrategy = NoShardStrategy.INSTANCE;
				}

				Configuration configuration = sqlSessionFactoryBean.getMainSqlSessionFactory().getConfiguration();
				MappedStatement mappedStatement = configuration.getMappedStatement(statement);
				BoundSql boundSql = mappedStatement.getBoundSql(wrapCollection(shardParam.getParams()));

				shardStrategy.setMainDataSource(sqlSessionFactoryBean.getMainDataSource());
				shardStrategy.setShardDataSources(sqlSessionFactoryBean.getShardDataSources());
				shardStrategy.setShardParam(shardParam);
				shardStrategy.setSql(boundSql.getSql());
				//
				StrategyHolder.setShardStrategy(shardStrategy);
				// 重新指定目标DataSource
				targetDS = shardStrategy.getTargetDataSource();
				SqlSessionTemplate sqlSessionTemplate = null;
				if (targetDS == null || (sqlSessionTemplate = dataSourceMap.get(targetDS)) == null) {
					targetDS = sqlSessionFactoryBean.getMainDataSource();
					sqlSessionTemplate = dataSourceMap.get(targetDS);
				}

				// 准备事务
				prepareTx(targetDS);

				return method.invoke(sqlSessionTemplate, args);
			} finally {
				StrategyHolder.removeShardStrategy();
			}
		}

		/**
		 * 、 准备事务
		 * 
		 * @param targetDS
		 */
		private void prepareTx(DataSource targetDS) {
			//
			TransactionHolder.setDataSource(targetDS);

			// for transaction
			TransactionInfoWrap txInfo = TransactionHolder.getTransactionInfo();
			if (txInfo != null) {
				TransactionAttribute attr = txInfo.getTransactionAttribute();
				if (attr != null) {
					createTxIfAbsent(targetDS, txInfo);
				}
			}
		}

		/**
		 * 如果不存在则创建事务
		 * 
		 * @param targetDS
		 * @param txInfo
		 */
		private void createTxIfAbsent(DataSource targetDS, TransactionInfoWrap txInfo) {
			Map<DataSource, LinkedList<TransactionInfoWrap>> txTree = TransactionHolder.getTxTree();
			if (txTree == null || !txTree.containsKey(targetDS)) {
				createTx(targetDS, txInfo);
			}
		}

		private void createTx(DataSource targetDS, TransactionInfoWrap txInfo) {
			TransactionStatus txStatus = txInfo.getTransactionManager()
					.getTransaction(txInfo.getTransactionAttribute());
			// txStatus = new TransactionStatusWrap((DefaultTransactionStatus)
			// txStatus);
			TransactionHolder.addStatusDS(txStatus, targetDS);
			//
			TransactionInfoWrap txInfoCopy = txInfo.newCopy();
			txInfoCopy.newTransactionStatus(txStatus);
			//
			TransactionHolder.addTxInfo2Tree(targetDS, txInfoCopy);
		}

		private Object wrapCollection(final Object object) {
			if (object instanceof List) {
				return new HashMap<String, Object>() {
					private static final long serialVersionUID = -2533602760878803345L;
					{
						put("list", object);
					}
				};
			} else if (object != null && object.getClass().isArray()) {
				return new HashMap<String, Object>() {
					private static final long serialVersionUID = 8371167260656531195L;
					{
						put("array", object);
					}
				};
			}
			return object;
		}
	}
}




(2)分表:通过自定义Configuration拦截器并传入分表参数来替换BoundSql的sql的方式实现分表策略

public class SqlSessionFactoryBean implements ApplicationContextAware, MultiDataSourceSupport {

	private final Logger logger = LoggerFactory.getLogger(getClass());

	private ApplicationContext applicationContext;

	private DataSource mainDataSource;

	private SqlSessionFactory mainSqlSessionFactory;

	private Map<String, DataSource> shardDataSources;

	private Map<String, SqlSessionFactory> shardSqlSessionFactory;

	private List<DataSource> shardDataSourceList;

	private Resource[] mapperLocations;

	private Map<String, ShardStrategy> shardStrategyMap = new HashMap<String, ShardStrategy>();
	private Map<String, Class<?>> shardStrategyConfig = new HashMap<String, Class<?>>();

	private SqlConverter sqlConverter = new DefaultSqlConverter();

	public DataSource getMainDataSource() {
		return mainDataSource;
	}

	public void setMainDataSource(DataSource mainDataSource) {
		if (mainDataSource instanceof TransactionAwareDataSourceProxy) {
			// If we got a TransactionAwareDataSourceProxy, we need to perform
			// transactions for its underlying target DataSource, else data
			// access code won't see properly exposed transactions (i.e.
			// transactions for the target DataSource).
			this.mainDataSource = ((TransactionAwareDataSourceProxy) mainDataSource).getTargetDataSource();
		} else {
			this.mainDataSource = mainDataSource;
		}
	}

	public void setShardDataSourceList(List<DataSource> shardDataSourceList) {
		this.shardDataSourceList = shardDataSourceList;
	}

	public Map<String, DataSource> getShardDataSources() {
		return shardDataSources;
	}

	public void setMapperLocations(Resource[] mapperLocations) {
		this.mapperLocations = mapperLocations;
	}

	public void setShardStrategy(Map<String, Class<?>> shardStrategyMap) {
		this.shardStrategyConfig = shardStrategyMap;
	}

	public SqlSessionFactory getMainSqlSessionFactory() {
		return mainSqlSessionFactory;
	}

	public Map<String, SqlSessionFactory> getShardSqlSessionFactory() {
		return shardSqlSessionFactory;
	}

	public Map<String, ShardStrategy> getShardStrategyMap() {
		return shardStrategyMap;
	}

	public void afterPropertiesSet() throws Exception {
		if (mainDataSource == null && (shardDataSourceList == null || shardDataSourceList.size() == 0)) {
			throw new RuntimeException(
					" Property 'mainDataSource' and property 'shardDataSourceList' can not be null together! ");
		}
		if (shardDataSourceList != null && shardDataSourceList.size() > 0) {
			shardDataSources = new LinkedHashMap<String, DataSource>();
			Map<String, DataSource> dataSourceMap = applicationContext.getBeansOfType(DataSource.class);
			for (Entry<String, DataSource> entry : dataSourceMap.entrySet()) {
				for (int i = 0; i < shardDataSourceList.size(); i++) {
					DataSource ds = shardDataSourceList.get(i);
					if (entry.getValue() == ds) {
						DataSource dataSource = entry.getValue();
						if (dataSource instanceof TransactionAwareDataSourceProxy) {
							dataSource = ((TransactionAwareDataSourceProxy) dataSource).getTargetDataSource();
						}
						shardDataSources.put(entry.getKey(), dataSource);
					}
				}
			}
		}
		if (mainDataSource == null) {
			if (shardDataSourceList.get(0) instanceof TransactionAwareDataSourceProxy) {
				this.mainDataSource = ((TransactionAwareDataSourceProxy) shardDataSourceList.get(0))
						.getTargetDataSource();
			} else {
				mainDataSource = shardDataSources.get(0);
			}
		}

		this.mainSqlSessionFactory = buildSqlSessionFactory(getMainDataSource());
		if (getShardDataSources() != null && getShardDataSources().size() > 0) {
			shardSqlSessionFactory = new LinkedHashMap<String, SqlSessionFactory>(getShardDataSources().size());
			for (Entry<String, DataSource> entry : getShardDataSources().entrySet()) {
				shardSqlSessionFactory.put(entry.getKey(), buildSqlSessionFactory(entry.getValue()));
			}
		}
		//
		if (shardStrategyConfig != null) {
			shardStrategyMap = new HashMap<String, ShardStrategy>();
			for (Map.Entry<String, Class<?>> entry : shardStrategyConfig.entrySet()) {
				Class<?> clazz = entry.getValue();
				if (!ShardStrategy.class.isAssignableFrom(clazz)) {
					throw new IllegalArgumentException("class " + clazz.getName()
							+ " is illegal, subclass of ShardStrategy is required.");
				}
				try {
					shardStrategyMap.put(entry.getKey(), (ShardStrategy) (entry.getValue().newInstance()));
				} catch (Exception e) {
					throw new RuntimeException("new instance for class " + clazz.getName() + " failed, error:"
							+ e.getMessage());
				}
			}
			//
			shardStrategyConfig = null;
		}
	}

	private SqlSessionFactory buildSqlSessionFactory(DataSource dataSource) throws IOException {
		ShardPlugin plugin = new ShardPlugin();
		plugin.setSqlConverter(sqlConverter);

		Configuration configuration = null;
		SpringManagedTransactionFactory transactionFactory = null;
		//
		configuration = new Configuration();
		configuration.addInterceptor(plugin);//通过自定义Configuration拦截器并传入分表参数来替换BoundSql的sql的方式实现分表策略
		//
		transactionFactory = new SpringManagedTransactionFactory(dataSource);

		Environment environment = new Environment(SqlSessionFactoryBean.class.getSimpleName(), transactionFactory,
				dataSource);
		configuration.setEnvironment(environment);

		if (!ObjectUtils.isEmpty(this.mapperLocations)) {
			for (Resource mapperLocation : this.mapperLocations) {
				if (mapperLocation == null) {
					continue;
				}
				// this block is a workaround for issue
				// http://code.google.com/p/mybatis/issues/detail?id=235
				// when running MyBatis 3.0.4. But not always works.
				// Not needed in 3.0.5 and above.
				String path;
				if (mapperLocation instanceof ClassPathResource) {
					path = ((ClassPathResource) mapperLocation).getPath();
				} else {
					path = mapperLocation.toString();
				}

				try {
					XMLMapperBuilder xmlMapperBuilder = new XMLMapperBuilder(mapperLocation.getInputStream(),
							configuration, path, configuration.getSqlFragments());
					xmlMapperBuilder.parse();
				} catch (Exception e) {
					throw new NestedIOException("Failed to parse mapping resource: '" + mapperLocation + "'", e);
				} finally {
					ErrorContext.instance().reset();
				}

				if (this.logger.isDebugEnabled()) {
					this.logger.debug("Parsed mapper file: '" + mapperLocation + "'");
				}
			}
		} else {
			if (this.logger.isDebugEnabled()) {
				this.logger.debug("Property 'mapperLocations' was not specified or no matching resources found");
			}
		}

		return new SqlSessionFactoryBuilder().build(configuration);
	}

	@Override
	public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
		this.applicationContext = applicationContext;
	}

}





3.为了便于理清思路,上传一张mybatis执行SQL的时序图:

 

  • 大小: 42.8 KB
分享到:
评论

相关推荐

    分库分表Sharding中间件Kamike.divide.zip

    开源个人参考淘宝的TDDL分库分表思路写的一个分库分表中间件Kamike.divide. 分库分表这个是8月份左右跟淘宝的数据分析部门的架构师离哲交流的时候产生的想法,离哲推荐采用TDDL进行分库分表。 回去一看,却...

    数据库分库分表思路.docx

    关系型数据库本身比较...当单表的数据量达到1000W或100G以后,由于查询维度较多,即使添加从库、优化索引,做很多操作时性能仍下降严重。此时就要考虑对其进行切分了,切分的目的就在于减少数据库的负担,缩短查询时间

    详解yii2实现分库分表的方案与思路

    主要介绍了利用yii2实现分库分表的方案与思路,在研究yii2如何分库分表之前,我先对yii2的核心概念和框架结构做了一个初步的探索,从而找到分库分表的思路。有需要的朋友可以参考借鉴,下面来一起看看吧。

    数据库分库分表存储分析思路

    解决大数据量存储时,分表存储时常见问题及解决方案

    分表分库的新思路——服务层Sharding框架

    分表分库的新思路——服务层Sharding框架,全SQL、全数据库兼容,ACID特性与原生数据库一致,能实现RR级别读写分离,无SQL解析性能更高

    Java并发编程及高并发解决方案

    第1章课程准备; 第2章并发基础. 第3章项目准备5 第4章线程安全性 第5章安全发布对象 第6章线程安全策略 第7章J.U.C之AQS ...第17章高并发之数据库切库分库分表思路 第18章高并发之高可用手段介绍 第19章课程总结

    数据库分库分表(sharding)系列

    数据库分库分表(sharding)实施策略图解1.准备阶段对数据库进行分库分表(Sharding化)前,需要开发人员充分了解系统业务逻辑和数据库schema.一个好的建议是绘制一张数据库ER图或领域模型图,以这类图为基础划分shard,...

    Mybaits-plus优雅的实现多数据源及分表

    不同的数据库写在对应的包名下分别进行扫描, 通过对id的取模运算实现分表的能力, 此代码仅只是提供思路,具体的实现方案还需要完善

    MYSQL数据库数据拆分之分库分表总结

    数据存储演进思路一:单库单表 单库单表是最常见的数据库设计,例如,有一张用户(user)表放在数据库db中,所有的用户都可以在db库中的user表中查到。 数据存储演进思路二:单库多表 随着用户数量的增加,user表的...

    sharding-jdbc按月分表样例

    sharind-jdbc样例,当中包含了按月动态分表,一个比较简单的demo,大家可以结合自己的需求进行修改,以及可以看下实现思路,仅供参考!

    高并发场景杂谈.zip

    为常见的高并发场景提供一些思路,从简单的分库分表到硬件级负载均衡都会涉及到,在不同的场景上选择不同的应对措施

    分片方法:分表分库的新思路-服务层共享框架,全SQL,全数据库兼容,ACID特性与原生数据库一致,能实现RR等级分离,无SQL解析性能更高

    我们目前国内主流的Sharding框架都是基于SQL来完成,其主要流程:是解析上层层次SQL结合对应的分表分库配置,对合并SQL进行改写并分发到对应的单机数据库上获得各个单机数据库的返回结果后,根据原SQL归并结果,返回...

    分布式架构系统生成全局唯一序列号的一个思路

    分布式架构下,唯一序列号生成是我们在设计一个系统,尤其是数据库使用分库分表的时候常常会遇见的问题。当分成若干个sharding表后,如何能够快速拿到一个唯一序列号,是经常遇到的问题。在携程账号数据库迁移MySQL...

    shardingJdbc.zip

    生产环境中,当单表数据达到一定的量级的时候,查询性能的瓶颈迟早会暴露出来,这个时候,解决思路可以考虑两种,一是从架构上对当前应用进行优化,比如使用es或mongodb等非关系型数据库...另一种就是分库分表 ......

    基于SSM的基层医疗服务的分析与设计.zip

    同时,还采用了分库分表技术,提高了系统的性能和可扩展性。系统安全性:本项目在开发过程中充分考虑了系统的安全性问题,采用了HTTPS加密传输协议、防止SQL注入攻击等措施,保障了用户数据的安全性和隐私保护。总之...

    基于电商业务中台最佳实践:总体架构介绍与交易业务中台核心设计

    天然的分库分表,消息解耦和分布式缓存设计,支持弹性扩容,以支持大数据高并发场景。接下来将分别介绍每个部分。中台部分在逻辑上分成了基础能力和平台产品两层,这样做的好处是,基础能力层聚焦于稳定收敛的业务...

Global site tag (gtag.js) - Google Analytics